From 2711bac41f701a34d4ab1a4ab4532d74fc97d2cd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 13:33:56 -0800 Subject: [PATCH 001/752] WIP --- examples/LLM_plus_GNN.py | 2 + torch_geometric/datasets/web_qsp_dataset.py | 131 ++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 examples/LLM_plus_GNN.py create mode 100644 torch_geometric/datasets/web_qsp_dataset.py diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py new file mode 100644 index 000000000000..23620ff464a6 --- /dev/null +++ b/examples/LLM_plus_GNN.py @@ -0,0 +1,2 @@ +# This example implements G-retriever +# https://github.com/XiaoxinHe/G-Retriever diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py new file mode 100644 index 000000000000..a92408786ae1 --- /dev/null +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -0,0 +1,131 @@ +import os.path as osp +import numpy as np +import torch + +from torch_geometric.data import InMemoryDataset +from datasets import load_dataset, concatenate_datasets +from transformers import AutoModel, AutoTokenizer + +class Sentence_Transformer(torch.nn.Module): + def __init__(self, pretrained_repo): + super(Sentence_Transformer, self).__init__() + print(f"inherit model weights from {pretrained_repo}") + self.bert_model = AutoModel.from_pretrained(pretrained_repo) + + def mean_pooling(self, model_output, attention_mask): + # First element of model_output contains all token embeddings + token_embeddings = model_output[0] + data_type = token_embeddings.dtype + input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).to(data_type) + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + def forward(self, input_ids, att_mask): + bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) + sentence_embeddings = self.mean_pooling(bert_out, att_mask) + sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) + return sentence_embeddings + +def sbert_text2embedding(model, tokenizer, device, text): + try: + encoding = tokenizer(text, padding=True, truncation=True, return_tensors='pt') + dataset = Dataset(input_ids=encoding.input_ids, attention_mask=encoding.attention_mask) + + # DataLoader + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False) + + # Placeholder for storing the embeddings + all_embeddings = [] + + # Iterate through batches + with torch.no_grad(): + + for batch in dataloader: + # Move batch to the appropriate device + batch = {key: value.to(device) for key, value in batch.items()} + + # Forward pass + embeddings = model(input_ids=batch["input_ids"], att_mask=batch["att_mask"]) + + # Append the embeddings to the list + all_embeddings.append(embeddings) + + # Concatenate the embeddings from all batches + all_embeddings = torch.cat(all_embeddings, dim=0).cpu() + except: + return torch.zeros((0, 1024)) + + return all_embeddings + +class WebQSPDataset(InMemoryDataset): + r""" + The WebQuestionsSP dataset was released as part of + “The Value of Semantic Parse Labeling for Knowledge + Base Question Answering” + [Yih, Richardson, Meek, Chang & Suh, 2016]. + It contains semantic parses, vs. answers, for a set of questions + that originally comes from WebQuestions [Berant et al., 2013]." + Processing based on "G-Retriever: Retrieval-Augmented Generation + for Textual Graph Understanding and Question Answering". + Args: + root (str): Root directory where the dataset should be saved. + force_reload (bool, optional): Whether to re-process the dataset. + (default: :obj:`False`) + """ + path = 'dataset/webqsp' + + def __init__( + self, + root: str, + force_reload: bool = False, + + ) -> None: + + super().__init__(root, None, None, + force_reload=force_reload) + self.load(self.processed_paths[0]) + self.prompt = 'Please answer the given question.' + self.graph = None + self.graph_type = 'Knowledge Graph' + self.model_name = 'sbert' + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + def download(self) -> None: + dataset = datasets.load_dataset("rmanluo/RoG-webqsp") + self.raw_dataset = datasets.concatenate_datasets([dataset['train'], dataset['validation'], dataset['test']]) + self.split_idxs = {'train':np.arange(len(dataset['train'])), 'val':np.arange(len(dataset['validation'])) + len(dataset['train']), 'test':np.arange(len(dataset['test'])) + len(dataset['train']) + len(dataset['validation'])} + + def process(self) -> None: + self.questions = [i['question'] for i in self.raw_dataset] + self.model = Sentence_Transformer(pretrained_repo) + self.model.to(self.device) + self.model.eval() + self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) + self.text2embedding = sbert_text2embedding + # encode questions + print('Encoding questions...') + self.q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) + print('Encoding graphs...') + self.list_of_graphs = [] + for index in tqdm(range(len(self.raw_dataset))): + raw_nodes = {} + raw_edges = [] + for tri in self.raw_dataset[i]['graph']: + h, r, t = tri + h = h.lower() + t = t.lower() + if h not in nodes: + raw_nodes[h] = len(raw_nodes) + if t not in nodes: + raw_nodes[t] = len(raw_nodes) + edges.append({'src': raw_nodes[h], 'edge_attr': r, 'dst': raw_nodes[t]}) + nodes = pd.DataFrame([{'node_id': v, 'node_attr': k} for k, v in nodes.items()], columns=['node_id', 'node_attr']) + edges = pd.DataFrame(edges, columns=['src', 'edge_attr', 'dst']) + # encode nodes + nodes.node_attr.fillna("", inplace=True) + x = self.text2embedding(self.model, self.tokenizer, self.device, nodes.node_attr.tolist()) + # encode edges + edge_attr = self.text2embedding(self.model, self.tokenizer, self.device, edges.edge_attr.tolist()) + edge_index = torch.LongTensor([edges.src.tolist(), edges.dst.tolist()]) + list_of_graphs.append(Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes), q_emb=self.q_embs[i])) + self.save(list_of_graphs, self.processed_paths[0]) + From 6f672e5006ea5d9a1dac730e14f29fca1fad331e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 13:58:24 -0800 Subject: [PATCH 002/752] WIP --- examples/LLM_plus_GNN.py | 375 ++++++++++++++++++++ torch_geometric/datasets/web_qsp_dataset.py | 2 +- 2 files changed, 376 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 23620ff464a6..5cad3e00947f 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,2 +1,377 @@ # This example implements G-retriever # https://github.com/XiaoxinHe/G-Retriever +# “G-Retriever significantly reduces hallucinations +# by 54% compared to the [LLM] baseline“ +import os +import gc +from tqdm import tqdm +import torch +from torch.utils.data import DataLoader +from torch.nn.utils import clip_grad_norm_ + +from src.model import load_model +from torch_geometric.datasets import WebQSPDataset +from src.utils.evaluate import eval_funcs +from torch_geometric import seed_everything +from src.utils.lr_schedule import adjust_learning_rate + + +BOS = '[INST]' +EOS_USER = '[/INST]' +EOS = '[/s]' +IGNORE_INDEX = -100 + + +def collate_fn(original_batch): + batch = {} + for k in original_batch[0].keys(): + batch[k] = [d[k] for d in original_batch] + if 'graph' in batch: + batch['graph'] = Batch.from_data_list(batch['graph']) + return batch + + +import contextlib +import torch +import torch.nn as nn +from torch.cuda.amp import autocast as autocast +from transformers import AutoModelForCausalLM, AutoTokenizer +from torch_scatter import scatter +from src.model.gnn import load_gnn_model +from peft import ( + LoraConfig, + get_peft_model, + prepare_model_for_int8_training, +) + + +class GraphLLM(torch.nn.Module): + + def __init__( + self, + args, + **kwargs + ): + super().__init__() + self.max_txt_len = args.max_txt_len + self.max_new_tokens = args.max_new_tokens + + print('Loading LLAMA') + kwargs = { + "max_memory": {0: '20GiB', 1: '20GiB', 2: '20GiB', 3: '20GiB'}, + "device_map": "auto", + "revision": "main", + } + + self.tokenizer = AutoTokenizer.from_pretrained(args.llm_model_path, use_fast=False, revision=kwargs["revision"]) + self.tokenizer.pad_token_id = 0 + self.tokenizer.padding_side = 'left' + + model = AutoModelForCausalLM.from_pretrained( + args.llm_model_path, + torch_dtype=torch.float16, + low_cpu_mem_usage=True, + **kwargs + ) + + if args.llm_frozen == 'True': + print("Freezing LLAMA!") + for name, param in model.named_parameters(): + param.requires_grad = False + else: + print("Training LLAMA with LORA!") + model = prepare_model_for_int8_training(model) + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + model = get_peft_model(model, config) + + self.model = model + print('Finish loading LLAMA!') + + self.graph_encoder = load_gnn_model[args.gnn_model_name]( + in_channels=args.gnn_in_dim, + out_channels=args.gnn_hidden_dim, + hidden_channels=args.gnn_hidden_dim, + num_layers=args.gnn_num_layers, + dropout=args.gnn_dropout, + num_heads=args.gnn_num_heads, + ).to(self.model.device) + + self.projector = nn.Sequential( + nn.Linear(args.gnn_hidden_dim, 2048), + nn.Sigmoid(), + nn.Linear(2048, 4096), + ).to(self.model.device) + + self.word_embedding = self.model.model.get_input_embeddings() + + @property + def device(self): + return list(self.parameters())[0].device + + def maybe_autocast(self, dtype=torch.bfloat16): + # if on cpu, don't use autocast + # if on gpu, use autocast with dtype if provided, otherwise use torch.float16 + enable_autocast = self.device != torch.device("cpu") + + if enable_autocast: + return torch.cuda.amp.autocast(dtype=dtype) + else: + return contextlib.nullcontext() + + def encode_graphs(self, samples): + graphs = samples['graph'] + graphs = graphs.to(self.model.device) + n_embeds, _ = self.graph_encoder(graphs.x, graphs.edge_index.long(), graphs.edge_attr) + + # mean pooling + g_embeds = scatter(n_embeds, graphs.batch, dim=0, reduce='mean') + + return g_embeds + + def forward(self, samples): + + # encode description, questions and labels + questions = self.tokenizer(samples["question"], add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) + labels = self.tokenizer(samples["label"], add_special_tokens=False) + + # encode special tokens + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) + bos_embeds = self.word_embedding(self.tokenizer(BOS, add_special_tokens=False, return_tensors='pt').input_ids[0]) + pad_embeds = self.word_embedding(torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + + # encode graphs + graph_embeds = self.encode_graphs(samples) + graph_embeds = self.projector(graph_embeds) + + batch_size = len(samples['id']) + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[i][:self.max_new_tokens] + eos_tokens.input_ids + input_ids = descriptions.input_ids[i][:self.max_txt_len] + questions.input_ids[i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding(torch.tensor(input_ids).to(self.model.device)) + inputs_embeds = torch.cat([bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) + + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX] * (inputs_embeds.shape[0]-len(label_input_ids))+label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length-batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat([pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0]*pad_length+batch_attention_mask[i] + batch_label_input_ids[i] = [IGNORE_INDEX] * pad_length+batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.model.device) + attention_mask = torch.tensor(batch_attention_mask).to(self.model.device) + label_input_ids = torch.tensor(batch_label_input_ids).to(self.model.device) + + with self.maybe_autocast(): + outputs = self.model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + + return outputs.loss + + def inference(self, samples): + + # encode description and questions + questions = self.tokenizer(samples["question"], add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) + + # encode special tokens + eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) + bos_embeds = self.word_embedding(self.tokenizer(BOS, add_special_tokens=False, return_tensors='pt').input_ids[0]) + pad_embeds = self.word_embedding(torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + + # encode graphs + graph_embeds = self.encode_graphs(samples) + graph_embeds = self.projector(graph_embeds) + + batch_size = len(samples['id']) + batch_inputs_embeds = [] + batch_attention_mask = [] + for i in range(batch_size): + # Add bos & eos token + input_ids = descriptions.input_ids[i][:self.max_txt_len] + questions.input_ids[i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding(torch.tensor(input_ids).to(self.model.device)) + inputs_embeds = torch.cat([bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length-batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat([pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0]*pad_length+batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.model.device) + attention_mask = torch.tensor(batch_attention_mask).to(self.model.device) + + with self.maybe_autocast(): + outputs = self.model.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=self.max_new_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return {'id': samples['id'], + 'pred': pred, + 'label': samples['label'], + 'question': samples['question'], + 'desc': samples['desc'], } + + def print_trainable_params(self): + trainable_params = 0 + all_param = 0 + + for _, param in self.named_parameters(): + num_params = param.numel() + + all_param += num_params + if param.requires_grad: + trainable_params += num_params + + return trainable_params, all_param + + +def main(): + seed_everything(42) + + dataset = WebQSPDataset() + idx_split = dataset.split_idxs + + # Step 2: Build Node Classification Dataset + train_dataset = [dataset[i] for i in idx_split['train']] + val_dataset = [dataset[i] for i in idx_split['val']] + test_dataset = [dataset[i] for i in idx_split['test']] + + train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, pin_memory=True, shuffle=True, collate_fn=collate_fn) + val_loader = DataLoader(val_dataset, batch_size=4, drop_last=False, pin_memory=True, shuffle=False, collate_fn=collate_fn) + test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, pin_memory=True, shuffle=False, collate_fn=collate_fn) + + # Step 3: Build Model + llm_model_path = "meta-llama/Llama-2-7b-chat-hf" + model = load_model[args.model_name](graph_type=dataset.graph_type, args=args, init_prompt=dataset.prompt) + + # Step 4 Set Optimizer + params = [p for _, p in model.named_parameters() if p.requires_grad] + optimizer = torch.optim.AdamW( + [{'params': params, 'lr': args.lr, 'weight_decay': args.wd}, ], + betas=(0.9, 0.95) + ) + trainable_params, all_param = model.print_trainable_params() + print(f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}") + + # Step 5. Training + num_training_steps = args.num_epochs * len(train_loader) + progress_bar = tqdm(range(num_training_steps)) + best_val_loss = float('inf') + + for epoch in range(args.num_epochs): + + model.train() + epoch_loss, accum_loss = 0., 0. + + for step, batch in enumerate(train_loader): + + optimizer.zero_grad() + loss = model(batch) + loss.backward() + + clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) + + if (step + 1) % args.grad_steps == 0: + adjust_learning_rate(optimizer.param_groups[0], args.lr, step / len(train_loader) + epoch, args) + + optimizer.step() + epoch_loss, accum_loss = epoch_loss + loss.item(), accum_loss + loss.item() + + if (step + 1) % args.grad_steps == 0: + lr = optimizer.param_groups[0]["lr"] + wandb.log({'Lr': lr}) + wandb.log({'Accum Loss': accum_loss / args.grad_steps}) + accum_loss = 0. + + progress_bar.update(1) + + print(f"Epoch: {epoch}|{args.num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") + wandb.log({'Train Loss (Epoch Mean)': epoch_loss / len(train_loader)}) + + val_loss = 0. + eval_output = [] + model.eval() + with torch.no_grad(): + for step, batch in enumerate(val_loader): + loss = model(batch) + val_loss += loss.item() + val_loss = val_loss/len(val_loader) + print(f"Epoch: {epoch}|{args.num_epochs}: Val Loss: {val_loss}") + wandb.log({'Val Loss': val_loss}) + + if val_loss < best_val_loss: + best_val_loss = val_loss + _save_checkpoint(model, optimizer, epoch, args, is_best=True) + best_epoch = epoch + + print(f'Epoch {epoch} Val Loss {val_loss} Best Val Loss {best_val_loss} Best Epoch {best_epoch}') + + if epoch - best_epoch >= args.patience: + print(f'Early stop at epoch {epoch}') + break + + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + + # Step 5. Evaluating + model = _reload_best_model(model, args) + model.eval() + eval_output = [] + progress_bar_test = tqdm(range(len(test_loader))) + for step, batch in enumerate(test_loader): + with torch.no_grad(): + output = model.inference(batch) + eval_output.append(output) + + progress_bar_test.update(1) + + # Step 6. Post-processing & compute metrics + os.makedirs(f'{args.output_dir}/{args.dataset}', exist_ok=True) + path = f'{args.output_dir}/{args.dataset}/model_name_{args.model_name}_llm_model_name_{args.llm_model_name}_llm_frozen_{args.llm_frozen}_max_txt_len_{args.max_txt_len}_max_new_tokens_{args.max_new_tokens}_gnn_model_name_{args.gnn_model_name}_patience_{args.patience}_num_epochs_{args.num_epochs}_seed{seed}.csv' + acc = eval_funcs[args.dataset](eval_output, path) + print(f'Test Acc {acc}') + + +if __name__ == "__main__": + main() + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() \ No newline at end of file diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a92408786ae1..f13435322b32 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,5 +1,4 @@ import os.path as osp -import numpy as np import torch from torch_geometric.data import InMemoryDataset @@ -66,6 +65,7 @@ class WebQSPDataset(InMemoryDataset): that originally comes from WebQuestions [Berant et al., 2013]." Processing based on "G-Retriever: Retrieval-Augmented Generation for Textual Graph Understanding and Question Answering". + Requires datasets and transformers from HuggingFace. Args: root (str): Root directory where the dataset should be saved. force_reload (bool, optional): Whether to re-process the dataset. From 5f2d747ad818b8b9d9addd1e26d14ff666619f2c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 16:55:01 -0800 Subject: [PATCH 003/752] WIP --- examples/LLM_plus_GNN.py | 141 +++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 59 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 5cad3e00947f..83d0fb47927c 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -11,10 +11,22 @@ from src.model import load_model from torch_geometric.datasets import WebQSPDataset -from src.utils.evaluate import eval_funcs +import pandas as pd from torch_geometric import seed_everything from src.utils.lr_schedule import adjust_learning_rate +import contextlib +import torch.nn as nn +from torch.cuda.amp import autocast as autocast +from transformers import AutoModelForCausalLM, AutoTokenizer +from torch_scatter import scatter +from src.model.gnn import load_gnn_model +from peft import ( + LoraConfig, + get_peft_model, + prepare_model_for_int8_training, +) + BOS = '[INST]' EOS_USER = '[/INST]' @@ -31,25 +43,52 @@ def collate_fn(original_batch): return batch -import contextlib -import torch -import torch.nn as nn -from torch.cuda.amp import autocast as autocast -from transformers import AutoModelForCausalLM, AutoTokenizer -from torch_scatter import scatter -from src.model.gnn import load_gnn_model -from peft import ( - LoraConfig, - get_peft_model, - prepare_model_for_int8_training, -) - - -class GraphLLM(torch.nn.Module): +def compute_accuracy(eval_output): + df = pd.concat([pd.DataFrame(d) for d in eval_output]) + all_hit = [] + all_precision = [] + all_recall = [] + all_f1 = [] + + for pred, label in zip(df.pred.tolist(), df.label.tolist()): + try: + pred = pred.split('[/s]')[0].strip().split('|') + hit = re.findall(pred[0], label) + all_hit.append(len(hit) > 0) + + label = label.split('|') + matches = set(pred).intersection(set(label)) + precision = len(matches)/len(set(label)) + recall = len(matches)/len(set(pred)) + if recall + precision == 0: + f1 = 0 + else: + f1 = 2 * precision * recall / (precision + recall) + + all_precision.append(precision) + all_recall.append(recall) + all_f1.append(f1) + + except: + print(f'Label: {label}') + print(f'Pred: {pred}') + print('------------------') + hit = sum(all_hit)/len(all_hit) + precision = sum(all_precision)/len(all_precision) + recall = sum(all_recall)/len(all_recall) + f1 = sum(all_f1)/len(all_f1) + + print(f'Hit: {hit:.4f}') + print(f'Precision: {precision:.4f}') + print(f'Recall: {recall:.4f}') + print(f'F1: {f1:.4f}') + + return hit + +class GAT_LLAMA(nn.Module): def __init__( self, - args, **kwargs ): super().__init__() @@ -62,46 +101,41 @@ def __init__( "device_map": "auto", "revision": "main", } - - self.tokenizer = AutoTokenizer.from_pretrained(args.llm_model_path, use_fast=False, revision=kwargs["revision"]) + llm_model_path = kwargs["path"] + self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) self.tokenizer.pad_token_id = 0 self.tokenizer.padding_side = 'left' model = AutoModelForCausalLM.from_pretrained( - args.llm_model_path, + llm_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True, **kwargs ) - if args.llm_frozen == 'True': - print("Freezing LLAMA!") - for name, param in model.named_parameters(): - param.requires_grad = False - else: - print("Training LLAMA with LORA!") - model = prepare_model_for_int8_training(model) - lora_r: int = 8 - lora_alpha: int = 16 - lora_dropout: float = 0.05 - lora_target_modules = [ - "q_proj", - "v_proj", - ] - config = LoraConfig( - r=lora_r, - lora_alpha=lora_alpha, - target_modules=lora_target_modules, - lora_dropout=lora_dropout, - bias="none", - task_type="CAUSAL_LM", - ) - model = get_peft_model(model, config) - - self.model = model + print("Training LLAMA with LORA!") + model = prepare_model_for_int8_training(model) + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + model = get_peft_model(model, config) + self.device = kwargs['device'] + self.model = model.to(self.device) print('Finish loading LLAMA!') - self.graph_encoder = load_gnn_model[args.gnn_model_name]( + self.graph_encoder = torch_geometric.nn.models.GAT( in_channels=args.gnn_in_dim, out_channels=args.gnn_hidden_dim, hidden_channels=args.gnn_hidden_dim, @@ -118,15 +152,10 @@ def __init__( self.word_embedding = self.model.model.get_input_embeddings() - @property - def device(self): - return list(self.parameters())[0].device - def maybe_autocast(self, dtype=torch.bfloat16): # if on cpu, don't use autocast # if on gpu, use autocast with dtype if provided, otherwise use torch.float16 enable_autocast = self.device != torch.device("cpu") - if enable_autocast: return torch.cuda.amp.autocast(dtype=dtype) else: @@ -143,7 +172,6 @@ def encode_graphs(self, samples): return g_embeds def forward(self, samples): - # encode description, questions and labels questions = self.tokenizer(samples["question"], add_special_tokens=False) descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) @@ -198,7 +226,6 @@ def forward(self, samples): return outputs.loss def inference(self, samples): - # encode description and questions questions = self.tokenizer(samples["question"], add_special_tokens=False) descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) @@ -252,7 +279,6 @@ def inference(self, samples): def print_trainable_params(self): trainable_params = 0 all_param = 0 - for _, param in self.named_parameters(): num_params = param.numel() @@ -280,7 +306,7 @@ def main(): # Step 3: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" - model = load_model[args.model_name](graph_type=dataset.graph_type, args=args, init_prompt=dataset.prompt) + model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, init_prompt=dataset.prompt, device=dataset.device) # Step 4 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -352,7 +378,6 @@ def main(): torch.cuda.reset_max_memory_allocated() # Step 5. Evaluating - model = _reload_best_model(model, args) model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) @@ -364,9 +389,7 @@ def main(): progress_bar_test.update(1) # Step 6. Post-processing & compute metrics - os.makedirs(f'{args.output_dir}/{args.dataset}', exist_ok=True) - path = f'{args.output_dir}/{args.dataset}/model_name_{args.model_name}_llm_model_name_{args.llm_model_name}_llm_frozen_{args.llm_frozen}_max_txt_len_{args.max_txt_len}_max_new_tokens_{args.max_new_tokens}_gnn_model_name_{args.gnn_model_name}_patience_{args.patience}_num_epochs_{args.num_epochs}_seed{seed}.csv' - acc = eval_funcs[args.dataset](eval_output, path) + acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') From bdc7b4a9a166017279001d6f7f6dd52f9c105dcc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 16:56:50 -0800 Subject: [PATCH 004/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 83d0fb47927c..aadcc2dce3fa 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,7 +1,7 @@ # This example implements G-retriever # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations -# by 54% compared to the [LLM] baseline“ +# by 54% compared to the [LLAMA] baseline“ import os import gc from tqdm import tqdm From 5a270411430c361e53654b791853b23de05dbf62 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 00:59:11 +0000 Subject: [PATCH 005/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 224 ++++++++++++-------- torch_geometric/datasets/web_qsp_dataset.py | 81 ++++--- 2 files changed, 193 insertions(+), 112 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index aadcc2dce3fa..fb0caee1bbf3 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -2,31 +2,26 @@ # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations # by 54% compared to the [LLAMA] baseline“ -import os +import contextlib import gc -from tqdm import tqdm -import torch -from torch.utils.data import DataLoader -from torch.nn.utils import clip_grad_norm_ +import os -from src.model import load_model -from torch_geometric.datasets import WebQSPDataset import pandas as pd -from torch_geometric import seed_everything -from src.utils.lr_schedule import adjust_learning_rate - -import contextlib +import torch import torch.nn as nn +from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training +from src.model import load_model +from src.model.gnn import load_gnn_model +from src.utils.lr_schedule import adjust_learning_rate from torch.cuda.amp import autocast as autocast -from transformers import AutoModelForCausalLM, AutoTokenizer +from torch.nn.utils import clip_grad_norm_ +from torch.utils.data import DataLoader from torch_scatter import scatter -from src.model.gnn import load_gnn_model -from peft import ( - LoraConfig, - get_peft_model, - prepare_model_for_int8_training, -) +from tqdm import tqdm +from transformers import AutoModelForCausalLM, AutoTokenizer +from torch_geometric import seed_everything +from torch_geometric.datasets import WebQSPDataset BOS = '[INST]' EOS_USER = '[/INST]' @@ -58,8 +53,8 @@ def compute_accuracy(eval_output): label = label.split('|') matches = set(pred).intersection(set(label)) - precision = len(matches)/len(set(label)) - recall = len(matches)/len(set(pred)) + precision = len(matches) / len(set(label)) + recall = len(matches) / len(set(pred)) if recall + precision == 0: f1 = 0 else: @@ -73,10 +68,10 @@ def compute_accuracy(eval_output): print(f'Label: {label}') print(f'Pred: {pred}') print('------------------') - hit = sum(all_hit)/len(all_hit) - precision = sum(all_precision)/len(all_precision) - recall = sum(all_recall)/len(all_recall) - f1 = sum(all_f1)/len(all_f1) + hit = sum(all_hit) / len(all_hit) + precision = sum(all_precision) / len(all_precision) + recall = sum(all_recall) / len(all_recall) + f1 = sum(all_f1) / len(all_f1) print(f'Hit: {hit:.4f}') print(f'Precision: {precision:.4f}') @@ -85,33 +80,34 @@ def compute_accuracy(eval_output): return hit -class GAT_LLAMA(nn.Module): - def __init__( - self, - **kwargs - ): +class GAT_LLAMA(nn.Module): + def __init__(self, **kwargs): super().__init__() self.max_txt_len = args.max_txt_len self.max_new_tokens = args.max_new_tokens print('Loading LLAMA') kwargs = { - "max_memory": {0: '20GiB', 1: '20GiB', 2: '20GiB', 3: '20GiB'}, + "max_memory": { + 0: '20GiB', + 1: '20GiB', + 2: '20GiB', + 3: '20GiB' + }, "device_map": "auto", "revision": "main", } llm_model_path = kwargs["path"] - self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) + self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, + use_fast=False) self.tokenizer.pad_token_id = 0 self.tokenizer.padding_side = 'left' - model = AutoModelForCausalLM.from_pretrained( - llm_model_path, - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - **kwargs - ) + model = AutoModelForCausalLM.from_pretrained(llm_model_path, + torch_dtype=torch.float16, + low_cpu_mem_usage=True, + **kwargs) print("Training LLAMA with LORA!") model = prepare_model_for_int8_training(model) @@ -164,7 +160,8 @@ def maybe_autocast(self, dtype=torch.bfloat16): def encode_graphs(self, samples): graphs = samples['graph'] graphs = graphs.to(self.model.device) - n_embeds, _ = self.graph_encoder(graphs.x, graphs.edge_index.long(), graphs.edge_attr) + n_embeds, _ = self.graph_encoder(graphs.x, graphs.edge_index.long(), + graphs.edge_attr) # mean pooling g_embeds = scatter(n_embeds, graphs.batch, dim=0, reduce='mean') @@ -173,15 +170,20 @@ def encode_graphs(self, samples): def forward(self, samples): # encode description, questions and labels - questions = self.tokenizer(samples["question"], add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) + questions = self.tokenizer(samples["question"], + add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], + add_special_tokens=False) labels = self.tokenizer(samples["label"], add_special_tokens=False) # encode special tokens eos_tokens = self.tokenizer(EOS, add_special_tokens=False) eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding(self.tokenizer(BOS, add_special_tokens=False, return_tensors='pt').input_ids[0]) - pad_embeds = self.word_embedding(torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + bos_embeds = self.word_embedding( + self.tokenizer(BOS, add_special_tokens=False, + return_tensors='pt').input_ids[0]) + pad_embeds = self.word_embedding( + torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -193,27 +195,41 @@ def forward(self, samples): batch_label_input_ids = [] for i in range(batch_size): # Add bos & eos token - label_input_ids = labels.input_ids[i][:self.max_new_tokens] + eos_tokens.input_ids - input_ids = descriptions.input_ids[i][:self.max_txt_len] + questions.input_ids[i] + eos_user_tokens.input_ids + label_input_ids - inputs_embeds = self.word_embedding(torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat([bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) + label_input_ids = labels.input_ids[ + i][:self.max_new_tokens] + eos_tokens.input_ids + input_ids = descriptions.input_ids[ + i][:self.max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.model.device)) + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) - label_input_ids = [IGNORE_INDEX] * (inputs_embeds.shape[0]-len(label_input_ids))+label_input_ids + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids batch_label_input_ids.append(label_input_ids) # pad inputs_embeds max_length = max([x.shape[0] for x in batch_inputs_embeds]) for i in range(batch_size): - pad_length = max_length-batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat([pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0]*pad_length+batch_attention_mask[i] - batch_label_input_ids[i] = [IGNORE_INDEX] * pad_length+batch_label_input_ids[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.model.device) - attention_mask = torch.tensor(batch_attention_mask).to(self.model.device) - label_input_ids = torch.tensor(batch_label_input_ids).to(self.model.device) + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.model.device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.model.device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.model.device) with self.maybe_autocast(): outputs = self.model( @@ -227,13 +243,18 @@ def forward(self, samples): def inference(self, samples): # encode description and questions - questions = self.tokenizer(samples["question"], add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], add_special_tokens=False) + questions = self.tokenizer(samples["question"], + add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], + add_special_tokens=False) # encode special tokens eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding(self.tokenizer(BOS, add_special_tokens=False, return_tensors='pt').input_ids[0]) - pad_embeds = self.word_embedding(torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + bos_embeds = self.word_embedding( + self.tokenizer(BOS, add_special_tokens=False, + return_tensors='pt').input_ids[0]) + pad_embeds = self.word_embedding( + torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -244,21 +265,30 @@ def inference(self, samples): batch_attention_mask = [] for i in range(batch_size): # Add bos & eos token - input_ids = descriptions.input_ids[i][:self.max_txt_len] + questions.input_ids[i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding(torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat([bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) + input_ids = descriptions.input_ids[ + i][:self.max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.model.device)) + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) # pad inputs_embeds max_length = max([x.shape[0] for x in batch_inputs_embeds]) for i in range(batch_size): - pad_length = max_length-batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat([pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0]*pad_length+batch_attention_mask[i] + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] - inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.model.device) - attention_mask = torch.tensor(batch_attention_mask).to(self.model.device) + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.model.device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.model.device) with self.maybe_autocast(): outputs = self.model.generate( @@ -270,11 +300,13 @@ def inference(self, samples): ) pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - return {'id': samples['id'], - 'pred': pred, - 'label': samples['label'], - 'question': samples['question'], - 'desc': samples['desc'], } + return { + 'id': samples['id'], + 'pred': pred, + 'label': samples['label'], + 'question': samples['question'], + 'desc': samples['desc'], + } def print_trainable_params(self): trainable_params = 0 @@ -300,22 +332,34 @@ def main(): val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] - train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, pin_memory=True, shuffle=True, collate_fn=collate_fn) - val_loader = DataLoader(val_dataset, batch_size=4, drop_last=False, pin_memory=True, shuffle=False, collate_fn=collate_fn) - test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, pin_memory=True, shuffle=False, collate_fn=collate_fn) + train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, + pin_memory=True, shuffle=True, + collate_fn=collate_fn) + val_loader = DataLoader(val_dataset, batch_size=4, drop_last=False, + pin_memory=True, shuffle=False, + collate_fn=collate_fn) + test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, + pin_memory=True, shuffle=False, + collate_fn=collate_fn) # Step 3: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" - model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, init_prompt=dataset.prompt, device=dataset.device) + model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, + init_prompt=dataset.prompt, device=dataset.device) # Step 4 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] - optimizer = torch.optim.AdamW( - [{'params': params, 'lr': args.lr, 'weight_decay': args.wd}, ], - betas=(0.9, 0.95) - ) + optimizer = torch.optim.AdamW([ + { + 'params': params, + 'lr': args.lr, + 'weight_decay': args.wd + }, + ], betas=(0.9, 0.95)) trainable_params, all_param = model.print_trainable_params() - print(f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}") + print( + f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" + ) # Step 5. Training num_training_steps = args.num_epochs * len(train_loader) @@ -336,10 +380,12 @@ def main(): clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) if (step + 1) % args.grad_steps == 0: - adjust_learning_rate(optimizer.param_groups[0], args.lr, step / len(train_loader) + epoch, args) + adjust_learning_rate(optimizer.param_groups[0], args.lr, + step / len(train_loader) + epoch, args) optimizer.step() - epoch_loss, accum_loss = epoch_loss + loss.item(), accum_loss + loss.item() + epoch_loss, accum_loss = epoch_loss + loss.item( + ), accum_loss + loss.item() if (step + 1) % args.grad_steps == 0: lr = optimizer.param_groups[0]["lr"] @@ -349,7 +395,9 @@ def main(): progress_bar.update(1) - print(f"Epoch: {epoch}|{args.num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") + print( + f"Epoch: {epoch}|{args.num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" + ) wandb.log({'Train Loss (Epoch Mean)': epoch_loss / len(train_loader)}) val_loss = 0. @@ -359,7 +407,7 @@ def main(): for step, batch in enumerate(val_loader): loss = model(batch) val_loss += loss.item() - val_loss = val_loss/len(val_loader) + val_loss = val_loss / len(val_loader) print(f"Epoch: {epoch}|{args.num_epochs}: Val Loss: {val_loss}") wandb.log({'Val Loss': val_loss}) @@ -368,7 +416,9 @@ def main(): _save_checkpoint(model, optimizer, epoch, args, is_best=True) best_epoch = epoch - print(f'Epoch {epoch} Val Loss {val_loss} Best Val Loss {best_val_loss} Best Epoch {best_epoch}') + print( + f'Epoch {epoch} Val Loss {val_loss} Best Val Loss {best_val_loss} Best Epoch {best_epoch}' + ) if epoch - best_epoch >= args.patience: print(f'Early stop at epoch {epoch}') @@ -397,4 +447,4 @@ def main(): main() torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() - gc.collect() \ No newline at end of file + gc.collect() diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f13435322b32..5f2b59f8864a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,9 +1,11 @@ import os.path as osp + import torch +from datasets import concatenate_datasets, load_dataset +from transformers import AutoModel, AutoTokenizer from torch_geometric.data import InMemoryDataset -from datasets import load_dataset, concatenate_datasets -from transformers import AutoModel, AutoTokenizer + class Sentence_Transformer(torch.nn.Module): def __init__(self, pretrained_repo): @@ -15,19 +17,25 @@ def mean_pooling(self, model_output, attention_mask): # First element of model_output contains all token embeddings token_embeddings = model_output[0] data_type = token_embeddings.dtype - input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).to(data_type) - return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + input_mask_expanded = attention_mask.unsqueeze(-1).expand( + token_embeddings.size()).to(data_type) + return torch.sum(token_embeddings * input_mask_expanded, + 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def forward(self, input_ids, att_mask): - bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) + bert_out = self.bert_model(input_ids=input_ids, + attention_mask=att_mask) sentence_embeddings = self.mean_pooling(bert_out, att_mask) sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) return sentence_embeddings + def sbert_text2embedding(model, tokenizer, device, text): try: - encoding = tokenizer(text, padding=True, truncation=True, return_tensors='pt') - dataset = Dataset(input_ids=encoding.input_ids, attention_mask=encoding.attention_mask) + encoding = tokenizer(text, padding=True, truncation=True, + return_tensors='pt') + dataset = Dataset(input_ids=encoding.input_ids, + attention_mask=encoding.attention_mask) # DataLoader dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False) @@ -43,7 +51,8 @@ def sbert_text2embedding(model, tokenizer, device, text): batch = {key: value.to(device) for key, value in batch.items()} # Forward pass - embeddings = model(input_ids=batch["input_ids"], att_mask=batch["att_mask"]) + embeddings = model(input_ids=batch["input_ids"], + att_mask=batch["att_mask"]) # Append the embeddings to the list all_embeddings.append(embeddings) @@ -55,9 +64,9 @@ def sbert_text2embedding(model, tokenizer, device, text): return all_embeddings + class WebQSPDataset(InMemoryDataset): - r""" - The WebQuestionsSP dataset was released as part of + r"""The WebQuestionsSP dataset was released as part of “The Value of Semantic Parse Labeling for Knowledge Base Question Answering” [Yih, Richardson, Meek, Chang & Suh, 2016]. @@ -66,33 +75,42 @@ class WebQSPDataset(InMemoryDataset): Processing based on "G-Retriever: Retrieval-Augmented Generation for Textual Graph Understanding and Question Answering". Requires datasets and transformers from HuggingFace. + Args: root (str): Root directory where the dataset should be saved. force_reload (bool, optional): Whether to re-process the dataset. (default: :obj:`False`) """ path = 'dataset/webqsp' - + def __init__( self, root: str, force_reload: bool = False, - ) -> None: - super().__init__(root, None, None, - force_reload=force_reload) + super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) self.prompt = 'Please answer the given question.' self.graph = None self.graph_type = 'Knowledge Graph' self.model_name = 'sbert' - self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") - self.raw_dataset = datasets.concatenate_datasets([dataset['train'], dataset['validation'], dataset['test']]) - self.split_idxs = {'train':np.arange(len(dataset['train'])), 'val':np.arange(len(dataset['validation'])) + len(dataset['train']), 'test':np.arange(len(dataset['test'])) + len(dataset['train']) + len(dataset['validation'])} + self.raw_dataset = datasets.concatenate_datasets( + [dataset['train'], dataset['validation'], dataset['test']]) + self.split_idxs = { + 'train': + np.arange(len(dataset['train'])), + 'val': + np.arange(len(dataset['validation'])) + len(dataset['train']), + 'test': + np.arange(len(dataset['test'])) + len(dataset['train']) + + len(dataset['validation']) + } def process(self) -> None: self.questions = [i['question'] for i in self.raw_dataset] @@ -103,7 +121,8 @@ def process(self) -> None: self.text2embedding = sbert_text2embedding # encode questions print('Encoding questions...') - self.q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) + self.q_embs = self.text2embedding(self.model, self.tokenizer, + self.device, self.questions) print('Encoding graphs...') self.list_of_graphs = [] for index in tqdm(range(len(self.raw_dataset))): @@ -117,15 +136,27 @@ def process(self) -> None: raw_nodes[h] = len(raw_nodes) if t not in nodes: raw_nodes[t] = len(raw_nodes) - edges.append({'src': raw_nodes[h], 'edge_attr': r, 'dst': raw_nodes[t]}) - nodes = pd.DataFrame([{'node_id': v, 'node_attr': k} for k, v in nodes.items()], columns=['node_id', 'node_attr']) + edges.append({ + 'src': raw_nodes[h], + 'edge_attr': r, + 'dst': raw_nodes[t] + }) + nodes = pd.DataFrame([{ + 'node_id': v, + 'node_attr': k + } for k, v in nodes.items()], columns=['node_id', 'node_attr']) edges = pd.DataFrame(edges, columns=['src', 'edge_attr', 'dst']) # encode nodes nodes.node_attr.fillna("", inplace=True) - x = self.text2embedding(self.model, self.tokenizer, self.device, nodes.node_attr.tolist()) + x = self.text2embedding(self.model, self.tokenizer, self.device, + nodes.node_attr.tolist()) # encode edges - edge_attr = self.text2embedding(self.model, self.tokenizer, self.device, edges.edge_attr.tolist()) - edge_index = torch.LongTensor([edges.src.tolist(), edges.dst.tolist()]) - list_of_graphs.append(Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes), q_emb=self.q_embs[i])) + edge_attr = self.text2embedding(self.model, self.tokenizer, + self.device, + edges.edge_attr.tolist()) + edge_index = torch.LongTensor( + [edges.src.tolist(), edges.dst.tolist()]) + list_of_graphs.append( + Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(nodes), q_emb=self.q_embs[i])) self.save(list_of_graphs, self.processed_paths[0]) - From d08c8a1cdf6b0447143d02641664999b685f3a61 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 17:16:25 -0800 Subject: [PATCH 006/752] WIP --- examples/LLM_plus_GNN.py | 72 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index fb0caee1bbf3..bd7bf8914516 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -2,6 +2,7 @@ # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations # by 54% compared to the [LLAMA] baseline“ + import contextlib import gc import os @@ -10,9 +11,7 @@ import torch import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training -from src.model import load_model -from src.model.gnn import load_gnn_model -from src.utils.lr_schedule import adjust_learning_rate +import math from torch.cuda.amp import autocast as autocast from torch.nn.utils import clip_grad_norm_ from torch.utils.data import DataLoader @@ -27,7 +26,18 @@ EOS_USER = '[/INST]' EOS = '[/s]' IGNORE_INDEX = -100 - +num_epochs = 10 + +def adjust_learning_rate(param_group, LR, epoch): + """Decay the learning rate with half-cycle cosine after warmup""" + min_lr = 5e-6 + warmup_epochs = 1 + if epoch < warmup_epochs: + lr = LR + else: + lr = min_lr + (LR - min_lr) * 0.5 * (1.0 + math.cos(math.pi * (epoch - warmup_epochs) / (num_epochs - warmup_epochs))) + param_group["lr"] = lr + return lr def collate_fn(original_batch): batch = {} @@ -84,8 +94,8 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): def __init__(self, **kwargs): super().__init__() - self.max_txt_len = args.max_txt_len - self.max_new_tokens = args.max_new_tokens + self.max_txt_len = 512 + self.max_new_tokens = 32 print('Loading LLAMA') kwargs = { @@ -132,16 +142,15 @@ def __init__(self, **kwargs): print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( - in_channels=args.gnn_in_dim, - out_channels=args.gnn_hidden_dim, - hidden_channels=args.gnn_hidden_dim, - num_layers=args.gnn_num_layers, - dropout=args.gnn_dropout, - num_heads=args.gnn_num_heads, + in_channels=1024, + out_channels=1024, + hidden_channels=1024, + num_layers=4, + num_heads=4, ).to(self.model.device) self.projector = nn.Sequential( - nn.Linear(args.gnn_hidden_dim, 2048), + nn.Linear(1024, 2048), nn.Sigmoid(), nn.Linear(2048, 4096), ).to(self.model.device) @@ -349,24 +358,26 @@ def main(): # Step 4 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] + lr = 1e-5 optimizer = torch.optim.AdamW([ { 'params': params, - 'lr': args.lr, - 'weight_decay': args.wd + 'lr': lr, + 'weight_decay': .05 }, ], betas=(0.9, 0.95)) + grad_steps = 2 trainable_params, all_param = model.print_trainable_params() print( f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" ) # Step 5. Training - num_training_steps = args.num_epochs * len(train_loader) + num_training_steps = num_epochs * len(train_loader) progress_bar = tqdm(range(num_training_steps)) best_val_loss = float('inf') - for epoch in range(args.num_epochs): + for epoch in range(num_epochs): model.train() epoch_loss, accum_loss = 0., 0. @@ -379,26 +390,25 @@ def main(): clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) - if (step + 1) % args.grad_steps == 0: - adjust_learning_rate(optimizer.param_groups[0], args.lr, - step / len(train_loader) + epoch, args) + if (step + 1) % grad_steps == 0: + adjust_learning_rate(optimizer.param_groups[0], lr, + step / len(train_loader) + epoch) optimizer.step() epoch_loss, accum_loss = epoch_loss + loss.item( ), accum_loss + loss.item() - if (step + 1) % args.grad_steps == 0: + if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] wandb.log({'Lr': lr}) - wandb.log({'Accum Loss': accum_loss / args.grad_steps}) + wandb.log({'Accum Loss': accum_loss / grad_steps}) accum_loss = 0. progress_bar.update(1) print( - f"Epoch: {epoch}|{args.num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" + f"Epoch: {epoch}|{num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" ) - wandb.log({'Train Loss (Epoch Mean)': epoch_loss / len(train_loader)}) val_loss = 0. eval_output = [] @@ -408,26 +418,18 @@ def main(): loss = model(batch) val_loss += loss.item() val_loss = val_loss / len(val_loader) - print(f"Epoch: {epoch}|{args.num_epochs}: Val Loss: {val_loss}") + print(f"Epoch: {epoch}|{num_epochs}: Val Loss: {val_loss}") wandb.log({'Val Loss': val_loss}) - if val_loss < best_val_loss: - best_val_loss = val_loss - _save_checkpoint(model, optimizer, epoch, args, is_best=True) - best_epoch = epoch - print( - f'Epoch {epoch} Val Loss {val_loss} Best Val Loss {best_val_loss} Best Epoch {best_epoch}' + f'Epoch {epoch} Val Loss {val_loss}' ) - if epoch - best_epoch >= args.patience: - print(f'Early stop at epoch {epoch}') - break - torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() # Step 5. Evaluating + print("Final Evaluation...") model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) From 6bc0dff1e306a9a4139b6925123b5b53b0e5531e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:17:34 +0000 Subject: [PATCH 007/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index bd7bf8914516..64670a73d588 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -5,13 +5,13 @@ import contextlib import gc +import math import os import pandas as pd import torch import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training -import math from torch.cuda.amp import autocast as autocast from torch.nn.utils import clip_grad_norm_ from torch.utils.data import DataLoader @@ -28,6 +28,7 @@ IGNORE_INDEX = -100 num_epochs = 10 + def adjust_learning_rate(param_group, LR, epoch): """Decay the learning rate with half-cycle cosine after warmup""" min_lr = 5e-6 @@ -35,10 +36,13 @@ def adjust_learning_rate(param_group, LR, epoch): if epoch < warmup_epochs: lr = LR else: - lr = min_lr + (LR - min_lr) * 0.5 * (1.0 + math.cos(math.pi * (epoch - warmup_epochs) / (num_epochs - warmup_epochs))) + lr = min_lr + (LR - min_lr) * 0.5 * ( + 1.0 + math.cos(math.pi * (epoch - warmup_epochs) / + (num_epochs - warmup_epochs))) param_group["lr"] = lr return lr + def collate_fn(original_batch): batch = {} for k in original_batch[0].keys(): @@ -421,9 +425,7 @@ def main(): print(f"Epoch: {epoch}|{num_epochs}: Val Loss: {val_loss}") wandb.log({'Val Loss': val_loss}) - print( - f'Epoch {epoch} Val Loss {val_loss}' - ) + print(f'Epoch {epoch} Val Loss {val_loss}') torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From 1c2c9bb2bfc85daaa5e6389a588086dfb5528b27 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 17:32:22 -0800 Subject: [PATCH 008/752] WIP --- examples/LLM_plus_GNN.py | 26 ++++++++------- torch_geometric/datasets/web_qsp_dataset.py | 35 ++++++++++++++++----- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 64670a73d588..67e1da614d6a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -9,6 +9,7 @@ import os import pandas as pd +import re import torch import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training @@ -18,9 +19,11 @@ from torch_scatter import scatter from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer +from torch_geometric.data import Batch from torch_geometric import seed_everything from torch_geometric.datasets import WebQSPDataset +import torch_geometric BOS = '[INST]' EOS_USER = '[/INST]' @@ -78,7 +81,7 @@ def compute_accuracy(eval_output): all_recall.append(recall) all_f1.append(f1) - except: + except: # noqa print(f'Label: {label}') print(f'Pred: {pred}') print('------------------') @@ -163,7 +166,8 @@ def __init__(self, **kwargs): def maybe_autocast(self, dtype=torch.bfloat16): # if on cpu, don't use autocast - # if on gpu, use autocast with dtype if provided, otherwise use torch.float16 + # if on gpu, use autocast with dtype if provided, + # otherwise use torch.float16 enable_autocast = self.device != torch.device("cpu") if enable_autocast: return torch.cuda.amp.autocast(dtype=dtype) @@ -373,18 +377,19 @@ def main(): grad_steps = 2 trainable_params, all_param = model.print_trainable_params() print( - f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" + f"trainable params: {trainable_params} || \ + all params: {all_param} || \ + trainable%: {100 * trainable_params / all_param}" ) # Step 5. Training num_training_steps = num_epochs * len(train_loader) progress_bar = tqdm(range(num_training_steps)) - best_val_loss = float('inf') for epoch in range(num_epochs): model.train() - epoch_loss, accum_loss = 0., 0. + epoch_loss = 0. for step, batch in enumerate(train_loader): @@ -399,19 +404,17 @@ def main(): step / len(train_loader) + epoch) optimizer.step() - epoch_loss, accum_loss = epoch_loss + loss.item( - ), accum_loss + loss.item() + epoch_loss = epoch_loss + loss.item( + ) if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] - wandb.log({'Lr': lr}) - wandb.log({'Accum Loss': accum_loss / grad_steps}) - accum_loss = 0. progress_bar.update(1) print( - f"Epoch: {epoch}|{num_epochs}: Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" + f"Epoch: {epoch}|{num_epochs}, \ + Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" ) val_loss = 0. @@ -423,7 +426,6 @@ def main(): val_loss += loss.item() val_loss = val_loss / len(val_loader) print(f"Epoch: {epoch}|{num_epochs}: Val Loss: {val_loss}") - wandb.log({'Val Loss': val_loss}) print(f'Epoch {epoch} Val Loss {val_loss}') diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5f2b59f8864a..822b1757b2f1 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,10 +1,30 @@ -import os.path as osp - import torch -from datasets import concatenate_datasets, load_dataset +import datasets from transformers import AutoModel, AutoTokenizer - +import torch.nn.functional as F from torch_geometric.data import InMemoryDataset +from torch.utils.data import DataLoader + + +class Dataset(torch.utils.data.Dataset): + def __init__(self, input_ids=None, attention_mask=None): + super().__init__() + self.data = { + "input_ids": input_ids, + "att_mask": attention_mask, + } + + def __len__(self): + return self.data["input_ids"].size(0) + + def __getitem__(self, index): + if isinstance(index, torch.Tensor): + index = index.item() + batch_data = dict() + for key in self.data.keys(): + if self.data[key] is not None: + batch_data[key] = self.data[key][index] + return batch_data class Sentence_Transformer(torch.nn.Module): @@ -38,7 +58,7 @@ def sbert_text2embedding(model, tokenizer, device, text): attention_mask=encoding.attention_mask) # DataLoader - dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False) + dataloader = DataLoader(dataset, batch_size=256, shuffle=False) # Placeholder for storing the embeddings all_embeddings = [] @@ -59,7 +79,8 @@ def sbert_text2embedding(model, tokenizer, device, text): # Concatenate the embeddings from all batches all_embeddings = torch.cat(all_embeddings, dim=0).cpu() - except: + except: # noqa + print("SBERT text embedding failed, returning 0s...") return torch.zeros((0, 1024)) return all_embeddings @@ -81,8 +102,6 @@ class WebQSPDataset(InMemoryDataset): force_reload (bool, optional): Whether to re-process the dataset. (default: :obj:`False`) """ - path = 'dataset/webqsp' - def __init__( self, root: str, From 046a91e2cd94b79ad036efbd71d609c5b940e2ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:33:23 +0000 Subject: [PATCH 009/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 23 ++++++++------------- torch_geometric/datasets/web_qsp_dataset.py | 9 ++++---- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 67e1da614d6a..0d0e0e990e4a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -7,9 +7,9 @@ import gc import math import os +import re import pandas as pd -import re import torch import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training @@ -19,11 +19,11 @@ from torch_scatter import scatter from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer -from torch_geometric.data import Batch +import torch_geometric from torch_geometric import seed_everything +from torch_geometric.data import Batch from torch_geometric.datasets import WebQSPDataset -import torch_geometric BOS = '[INST]' EOS_USER = '[/INST]' @@ -81,7 +81,7 @@ def compute_accuracy(eval_output): all_recall.append(recall) all_f1.append(f1) - except: # noqa + except: # noqa print(f'Label: {label}') print(f'Pred: {pred}') print('------------------') @@ -376,11 +376,9 @@ def main(): ], betas=(0.9, 0.95)) grad_steps = 2 trainable_params, all_param = model.print_trainable_params() - print( - f"trainable params: {trainable_params} || \ + print(f"trainable params: {trainable_params} || \ all params: {all_param} || \ - trainable%: {100 * trainable_params / all_param}" - ) + trainable%: {100 * trainable_params / all_param}") # Step 5. Training num_training_steps = num_epochs * len(train_loader) @@ -404,18 +402,15 @@ def main(): step / len(train_loader) + epoch) optimizer.step() - epoch_loss = epoch_loss + loss.item( - ) + epoch_loss = epoch_loss + loss.item() if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] progress_bar.update(1) - print( - f"Epoch: {epoch}|{num_epochs}, \ - Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}" - ) + print(f"Epoch: {epoch}|{num_epochs}, \ + Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") val_loss = 0. eval_output = [] diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 822b1757b2f1..b839b673d53b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,9 +1,10 @@ -import torch import datasets -from transformers import AutoModel, AutoTokenizer +import torch import torch.nn.functional as F -from torch_geometric.data import InMemoryDataset from torch.utils.data import DataLoader +from transformers import AutoModel, AutoTokenizer + +from torch_geometric.data import InMemoryDataset class Dataset(torch.utils.data.Dataset): @@ -79,7 +80,7 @@ def sbert_text2embedding(model, tokenizer, device, text): # Concatenate the embeddings from all batches all_embeddings = torch.cat(all_embeddings, dim=0).cpu() - except: # noqa + except: # noqa print("SBERT text embedding failed, returning 0s...") return torch.zeros((0, 1024)) From 2194d47bce318459502c0182ad108ad0cee6c319 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 17:47:21 -0800 Subject: [PATCH 010/752] WIP --- examples/LLM_plus_GNN.py | 2 -- torch_geometric/datasets/web_qsp_dataset.py | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 67e1da614d6a..c3ca5e5d7447 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -6,14 +6,12 @@ import contextlib import gc import math -import os import pandas as pd import re import torch import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training -from torch.cuda.amp import autocast as autocast from torch.nn.utils import clip_grad_norm_ from torch.utils.data import DataLoader from torch_scatter import scatter diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 822b1757b2f1..e2382afbac41 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -4,7 +4,8 @@ import torch.nn.functional as F from torch_geometric.data import InMemoryDataset from torch.utils.data import DataLoader - +import tqdm +import pandas as pd class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids=None, attention_mask=None): @@ -123,16 +124,17 @@ def download(self) -> None: [dataset['train'], dataset['validation'], dataset['test']]) self.split_idxs = { 'train': - np.arange(len(dataset['train'])), + torch.arange(len(dataset['train'])), 'val': - np.arange(len(dataset['validation'])) + len(dataset['train']), + torch.arange(len(dataset['validation'])) + len(dataset['train']), 'test': - np.arange(len(dataset['test'])) + len(dataset['train']) + + torch.arange(len(dataset['test'])) + len(dataset['train']) + len(dataset['validation']) } def process(self) -> None: self.questions = [i['question'] for i in self.raw_dataset] + pretrained_repo = 'sentence-transformers/all-roberta-large-v1' self.model = Sentence_Transformer(pretrained_repo) self.model.to(self.device) self.model.eval() @@ -143,7 +145,7 @@ def process(self) -> None: self.q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print('Encoding graphs...') - self.list_of_graphs = [] + list_of_graphs = [] for index in tqdm(range(len(self.raw_dataset))): raw_nodes = {} raw_edges = [] @@ -151,11 +153,11 @@ def process(self) -> None: h, r, t = tri h = h.lower() t = t.lower() - if h not in nodes: + if h not in raw_nodes: raw_nodes[h] = len(raw_nodes) - if t not in nodes: + if t not in raw_nodes: raw_nodes[t] = len(raw_nodes) - edges.append({ + raw_edges.append({ 'src': raw_nodes[h], 'edge_attr': r, 'dst': raw_nodes[t] From 14f1d85dae7d6ae61cea6fd4e78e6e853a215df2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:48:58 +0000 Subject: [PATCH 011/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 2 ++ torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 5a21305c56ce..0b8f66680d08 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -6,10 +6,12 @@ import contextlib import gc import math + <<<<<<< HEAD ======= import os import re + >>>>>>> 046a91e2cd94b79ad036efbd71d609c5b940e2ad import pandas as pd diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ac2bccf5deb0..1848a4067d2c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,13 +1,14 @@ import datasets +import pandas as pd import torch import torch.nn.functional as F -from torch.utils.data import DataLoader import tqdm -import pandas as pd +from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer from torch_geometric.data import InMemoryDataset + class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids=None, attention_mask=None): super().__init__() From 503f7aabd86bc5e04218dbe1d0ad031fd2f2da42 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Feb 2024 17:56:58 -0800 Subject: [PATCH 012/752] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5447099b52..7c207a28cc22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added ### Changed - +- Added the `WebQSPDataset` dataset with example training G-Retriever (GNN+LLM) ([#8984](https://github.com/pyg-team/pytorch_geometric/pull/8984)) - Added approximate `faiss`-based KNN-search ([#8952](https://github.com/pyg-team/pytorch_geometric/pull/8952)) - Breaking Change: Added support for `EdgeIndex` in `cugraph` GNN layers ([#8938](https://github.com/pyg-team/pytorch_geometric/pull/8937)) - Added the `dim` arg to `torch.cross` calls ([#8918](https://github.com/pyg-team/pytorch_geometric/pull/8918)) From 5e3b4008050f810f1aa03ccfe7e16f89ba57268b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:58:02 +0000 Subject: [PATCH 013/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c207a28cc22..cbe301c2aeee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added ### Changed + - Added the `WebQSPDataset` dataset with example training G-Retriever (GNN+LLM) ([#8984](https://github.com/pyg-team/pytorch_geometric/pull/8984)) - Added approximate `faiss`-based KNN-search ([#8952](https://github.com/pyg-team/pytorch_geometric/pull/8952)) - Breaking Change: Added support for `EdgeIndex` in `cugraph` GNN layers ([#8938](https://github.com/pyg-team/pytorch_geometric/pull/8937)) From 4e9b48bd0c55e397c0637a2bf47367d2d15ca215 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 18:00:25 -0800 Subject: [PATCH 014/752] WIP --- examples/LLM_plus_GNN.py | 4 ---- torch_geometric/datasets/web_qsp_dataset.py | 10 +++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 5a21305c56ce..f5611401d3f4 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -6,11 +6,7 @@ import contextlib import gc import math -<<<<<<< HEAD -======= -import os import re ->>>>>>> 046a91e2cd94b79ad036efbd71d609c5b940e2ad import pandas as pd import torch diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ac2bccf5deb0..ec268e4e8759 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -6,7 +6,7 @@ import pandas as pd from transformers import AutoModel, AutoTokenizer -from torch_geometric.data import InMemoryDataset +from torch_geometric.data import InMemoryDataset, Data class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids=None, attention_mask=None): @@ -150,7 +150,7 @@ def process(self) -> None: for index in tqdm(range(len(self.raw_dataset))): raw_nodes = {} raw_edges = [] - for tri in self.raw_dataset[i]['graph']: + for tri in self.raw_dataset[index]['graph']: h, r, t = tri h = h.lower() t = t.lower() @@ -166,8 +166,8 @@ def process(self) -> None: nodes = pd.DataFrame([{ 'node_id': v, 'node_attr': k - } for k, v in nodes.items()], columns=['node_id', 'node_attr']) - edges = pd.DataFrame(edges, columns=['src', 'edge_attr', 'dst']) + } for k, v in raw_nodes.items()], columns=['node_id', 'node_attr']) + edges = pd.DataFrame(raw_edges, columns=['src', 'edge_attr', 'dst']) # encode nodes nodes.node_attr.fillna("", inplace=True) x = self.text2embedding(self.model, self.tokenizer, self.device, @@ -180,5 +180,5 @@ def process(self) -> None: [edges.src.tolist(), edges.dst.tolist()]) list_of_graphs.append( Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(nodes), q_emb=self.q_embs[i])) + num_nodes=len(nodes), q_emb=self.q_embs[index])) self.save(list_of_graphs, self.processed_paths[0]) From 5d86b1f4e0b2d2013a3bf83f550e755774975efa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 02:01:55 +0000 Subject: [PATCH 015/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 856f92ff4e1c..7a643d50c860 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -6,7 +6,7 @@ from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer -from torch_geometric.data import InMemoryDataset, Data +from torch_geometric.data import Data, InMemoryDataset class Dataset(torch.utils.data.Dataset): @@ -168,7 +168,8 @@ def process(self) -> None: 'node_id': v, 'node_attr': k } for k, v in raw_nodes.items()], columns=['node_id', 'node_attr']) - edges = pd.DataFrame(raw_edges, columns=['src', 'edge_attr', 'dst']) + edges = pd.DataFrame(raw_edges, + columns=['src', 'edge_attr', 'dst']) # encode nodes nodes.node_attr.fillna("", inplace=True) x = self.text2embedding(self.model, self.tokenizer, self.device, From 604c81912b2cf8a3c96aac8b87a7058f34ee896a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 18:02:34 -0800 Subject: [PATCH 016/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f5611401d3f4..b8d759432f24 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -31,7 +31,7 @@ def adjust_learning_rate(param_group, LR, epoch): - """Decay the learning rate with half-cycle cosine after warmup""" + # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 warmup_epochs = 1 if epoch < warmup_epochs: From 739c47ad5988ee448b63ae7c11b973c4a886b177 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 18:14:54 -0800 Subject: [PATCH 017/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index b8d759432f24..ba5c9faa9c7d 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -14,7 +14,7 @@ from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training from torch.nn.utils import clip_grad_norm_ from torch.utils.data import DataLoader -from torch_scatter import scatter +from torch_geometric.utils import scatter from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer From e70425f9d887c7fe95d9d940826438c5c2a12ff5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 02:15:59 +0000 Subject: [PATCH 018/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index ba5c9faa9c7d..fbc995fa4d5d 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -14,7 +14,6 @@ from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training from torch.nn.utils import clip_grad_norm_ from torch.utils.data import DataLoader -from torch_geometric.utils import scatter from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer @@ -22,6 +21,7 @@ from torch_geometric import seed_everything from torch_geometric.data import Batch from torch_geometric.datasets import WebQSPDataset +from torch_geometric.utils import scatter BOS = '[INST]' EOS_USER = '[/INST]' From 4bc2c6ee084b1d8248659bf2b39fec55caac415a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 18:17:01 -0800 Subject: [PATCH 019/752] WIP --- torch_geometric/datasets/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index b2bbd3b3181b..d5ffe5bf8bdc 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -76,6 +76,7 @@ from .wikidata import Wikidata5M from .myket import MyketDataset from .brca_tgca import BrcaTcga +from .web_qsp_dataset import WebQSPDataset from .dbp15k import DBP15K from .aminer import AMiner @@ -107,6 +108,7 @@ import torch_geometric.datasets.utils homo_datasets = [ + 'WebQSPDataset' 'KarateClub', 'TUDataset', 'GNNBenchmarkDataset', From dbc3278141ab93afdf6fe1e477d9e7658d9671ef Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Feb 2024 18:19:34 -0800 Subject: [PATCH 020/752] Update README.md --- examples/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/README.md b/examples/README.md index 25fe7a7cb66d..a4213a14dbc9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,8 @@ For examples on [Open Graph Benchmark](https://ogb.stanford.edu/) datasets, see - [`ogbn_proteins_deepgcn.py`](./ogbn_proteins_deepgcn.py) is an example to showcase how to train deep GNNs on the `ogbn-proteins` dataset. - [`ogbn_papers_100m.py`](./ogbn_papers_100m.py) is an example for training a GNN on the large-scale `ogbn-papers100m` dataset, containing approximately ~1.6B edges. +For an example on co-training a GNN with an LLM, see [`examples/LLM_plus_GNN.py`](./LLM_plus_GNN.py). + For examples on using `torch.compile`, see the examples under [`examples/compile`](./compile). For examples on scaling PyG up via multi-GPUs, see the examples under [`examples/multi_gpu`](./multi_gpu). From 261b5f954e67858e872171936e4c2d3479ed0a55 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 27 Feb 2024 18:22:22 -0800 Subject: [PATCH 021/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 7a643d50c860..00488da475e5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -107,7 +107,7 @@ class WebQSPDataset(InMemoryDataset): """ def __init__( self, - root: str, + root: str = '', force_reload: bool = False, ) -> None: From 5c59b25a700fc1b8651fad32dd3cac131d6082fc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 28 Feb 2024 13:13:55 -0800 Subject: [PATCH 022/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index fbc995fa4d5d..4586fc327a55 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,7 +1,7 @@ # This example implements G-retriever # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations -# by 54% compared to the [LLAMA] baseline“ +# by 54% compared to the [LLM] baseline“ import contextlib import gc From 2f178d1ae3ad6c91b2afa9df47512932379d4094 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 28 Feb 2024 15:32:19 -0800 Subject: [PATCH 023/752] WIP --- examples/LLM_plus_GNN.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 4586fc327a55..5e193f441b6e 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,7 +1,10 @@ # This example implements G-retriever # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations -# by 54% compared to the [LLM] baseline“ +# by 54% compared to the [LLM] baseline“. +# Original work uses LLAMA2 from Meta for LLM. +# This example uses GEMMA from google for the LLM +# as this is the current open source SOTA LLM. import contextlib import gc From 751cdbe0e5265938c912fe4219748260922875f5 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 29 Feb 2024 09:12:31 -0800 Subject: [PATCH 024/752] Update README.md --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index a4213a14dbc9..17c926d4adec 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,7 +13,7 @@ For examples on [Open Graph Benchmark](https://ogb.stanford.edu/) datasets, see - [`ogbn_proteins_deepgcn.py`](./ogbn_proteins_deepgcn.py) is an example to showcase how to train deep GNNs on the `ogbn-proteins` dataset. - [`ogbn_papers_100m.py`](./ogbn_papers_100m.py) is an example for training a GNN on the large-scale `ogbn-papers100m` dataset, containing approximately ~1.6B edges. -For an example on co-training a GNN with an LLM, see [`examples/LLM_plus_GNN.py`](./LLM_plus_GNN.py). +For an example on co-training a LLM with a GNN by using both textual and knowledge graph data, see [`examples/LLM_plus_GNN.py`](./LLM_plus_GNN.py). For examples on using `torch.compile`, see the examples under [`examples/compile`](./compile). From 599a843b2b5a2257e3f55cfe838b1dd1fbad6e85 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:04:00 -0800 Subject: [PATCH 025/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 00488da475e5..8312b6ff0e5a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -120,6 +120,10 @@ def __init__( self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') + @property + def raw_file_names(self) -> List[str]: + return [] + def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( From 5cdeececa469a417b286c7791ec61ca79bb5a5ec Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:06:44 -0800 Subject: [PATCH 026/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 8312b6ff0e5a..c1aadbc171bf 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -5,6 +5,7 @@ import tqdm from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer +from typing import List from torch_geometric.data import Data, InMemoryDataset From b089e5299dcc8304d7c716f02290d6818f202928 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:10:25 -0800 Subject: [PATCH 027/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index c1aadbc171bf..dd39ab5c0170 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -125,6 +125,10 @@ def __init__( def raw_file_names(self) -> List[str]: return [] + @property + def processed_file_names(self) -> List[str]: + return [] + def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( From 31185393b8a9758723a1f909d6d1a282f5c26d3e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:12:32 -0800 Subject: [PATCH 028/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index dd39ab5c0170..c78b5c9e1aef 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -113,13 +113,14 @@ def __init__( ) -> None: super().__init__(root, None, None, force_reload=force_reload) - self.load(self.processed_paths[0]) self.prompt = 'Please answer the given question.' self.graph = None self.graph_type = 'Knowledge Graph' self.model_name = 'sbert' self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') + self.load(self.processed_paths[0]) + @property def raw_file_names(self) -> List[str]: From 560c21f8f8562437dfc02ee2f7aec18b931e742b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:17:16 -0800 Subject: [PATCH 029/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index c78b5c9e1aef..2796d6e3ad2f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -112,13 +112,13 @@ def __init__( force_reload: bool = False, ) -> None: - super().__init__(root, None, None, force_reload=force_reload) self.prompt = 'Please answer the given question.' self.graph = None self.graph_type = 'Knowledge Graph' self.model_name = 'sbert' self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') + super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) From 4e12b5884553df2b8abab65f0a4e27c38fc5f936 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:21:50 -0800 Subject: [PATCH 030/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2796d6e3ad2f..e7249587f4cf 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -158,7 +158,7 @@ def process(self) -> None: self.device, self.questions) print('Encoding graphs...') list_of_graphs = [] - for index in tqdm(range(len(self.raw_dataset))): + for index in tqdm(range(len(self.questions))): raw_nodes = {} raw_edges = [] for tri in self.raw_dataset[index]['graph']: From 8a9620b5d5923800477e73af3f929909cb8dec1b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:24:36 -0800 Subject: [PATCH 031/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e7249587f4cf..6b45add57b37 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -2,7 +2,7 @@ import pandas as pd import torch import torch.nn.functional as F -import tqdm +from tqdm import tqdm from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer from typing import List From 3d7c2fe5de6f188c1aaae229d3a3c740c1ce3904 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 10:28:11 -0800 Subject: [PATCH 032/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 6b45add57b37..31d230e01f6b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -128,7 +128,7 @@ def raw_file_names(self) -> List[str]: @property def processed_file_names(self) -> List[str]: - return [] + return ['list_of_graphs.pt'] def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") From 2d36b6e1f7506d3db9c6506b53bbe6e5db8ce816 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 11:53:44 -0800 Subject: [PATCH 033/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 31d230e01f6b..423d23023501 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -84,7 +84,7 @@ def sbert_text2embedding(model, tokenizer, device, text): # Concatenate the embeddings from all batches all_embeddings = torch.cat(all_embeddings, dim=0).cpu() except: # noqa - print("SBERT text embedding failed, returning 0s...") + print("SBERT text embedding failed, returning torch.zeros((0, 1024))...") return torch.zeros((0, 1024)) return all_embeddings From 224257e5bc3dcb18bb8297d2ee55cd05f4692256 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 12:38:09 -0800 Subject: [PATCH 034/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 423d23023501..a7daaeb149d7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -158,7 +158,7 @@ def process(self) -> None: self.device, self.questions) print('Encoding graphs...') list_of_graphs = [] - for index in tqdm(range(len(self.questions))): + #for index in tqdm(range(len(self.questions))): raw_nodes = {} raw_edges = [] for tri in self.raw_dataset[index]['graph']: @@ -193,4 +193,5 @@ def process(self) -> None: list_of_graphs.append( Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes), q_emb=self.q_embs[index])) + print("data[" + str(i) + "] =", data) self.save(list_of_graphs, self.processed_paths[0]) From 83e082301ca2fdeab13c66008104d712f60721d0 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 12:40:24 -0800 Subject: [PATCH 035/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a7daaeb149d7..40fbc08574b2 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -159,6 +159,7 @@ def process(self) -> None: print('Encoding graphs...') list_of_graphs = [] #for index in tqdm(range(len(self.questions))): + for index in range(len(self.questions)): raw_nodes = {} raw_edges = [] for tri in self.raw_dataset[index]['graph']: From 86a1958d747e0e6f96cc2121a8149caad853df5c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 13:52:52 -0800 Subject: [PATCH 036/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 40fbc08574b2..2aa23fba6c27 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -145,17 +145,13 @@ def download(self) -> None: } def process(self) -> None: - self.questions = [i['question'] for i in self.raw_dataset] + self.labels = pretrained_repo = 'sentence-transformers/all-roberta-large-v1' self.model = Sentence_Transformer(pretrained_repo) self.model.to(self.device) self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding - # encode questions - print('Encoding questions...') - self.q_embs = self.text2embedding(self.model, self.tokenizer, - self.device, self.questions) print('Encoding graphs...') list_of_graphs = [] #for index in tqdm(range(len(self.questions))): @@ -193,6 +189,9 @@ def process(self) -> None: [edges.src.tolist(), edges.dst.tolist()]) list_of_graphs.append( Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(nodes), q_emb=self.q_embs[index])) - print("data[" + str(i) + "] =", data) + num_nodes=len(nodes), + question=raw_dataset[index]["question"], + label=raw_dataset[index]["label"], + desc=raw_dataset[index]["desc"]) + print("data[" + str(index) + "] =", data) self.save(list_of_graphs, self.processed_paths[0]) From 345460e3fd7b06d9c432bce89d737b378c20b019 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 13:56:30 -0800 Subject: [PATCH 037/752] WIP --- examples/LLM_plus_GNN.py | 3 ++- torch_geometric/datasets/web_qsp_dataset.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 5e193f441b6e..808979321a33 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -16,7 +16,7 @@ import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training from torch.nn.utils import clip_grad_norm_ -from torch.utils.data import DataLoader +from torch_geometric.data import DataLoader from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer @@ -187,6 +187,7 @@ def encode_graphs(self, samples): return g_embeds def forward(self, samples): + print("samples=", samples) # encode description, questions and labels questions = self.tokenizer(samples["question"], add_special_tokens=False) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2aa23fba6c27..8df148bc5fc5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -145,14 +145,12 @@ def download(self) -> None: } def process(self) -> None: - self.labels = pretrained_repo = 'sentence-transformers/all-roberta-large-v1' self.model = Sentence_Transformer(pretrained_repo) self.model.to(self.device) self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding - print('Encoding graphs...') list_of_graphs = [] #for index in tqdm(range(len(self.questions))): for index in range(len(self.questions)): From f04a039603c15dcccfbd2647d11907c667667f91 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 13:57:16 -0800 Subject: [PATCH 038/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 8df148bc5fc5..2856b9384f68 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -190,6 +190,6 @@ def process(self) -> None: num_nodes=len(nodes), question=raw_dataset[index]["question"], label=raw_dataset[index]["label"], - desc=raw_dataset[index]["desc"]) + desc=raw_dataset[index]["desc"])) print("data[" + str(index) + "] =", data) self.save(list_of_graphs, self.processed_paths[0]) From 0a47ad03118d75f658d061cd45697e23ece521b8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 14:00:40 -0800 Subject: [PATCH 039/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2856b9384f68..74334aebbb0e 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -153,10 +153,10 @@ def process(self) -> None: self.text2embedding = sbert_text2embedding list_of_graphs = [] #for index in tqdm(range(len(self.questions))): - for index in range(len(self.questions)): + for index, data_i in enumerate(self.raw_dataset): raw_nodes = {} raw_edges = [] - for tri in self.raw_dataset[index]['graph']: + for tri in data_i['graph']: h, r, t = tri h = h.lower() t = t.lower() @@ -188,8 +188,8 @@ def process(self) -> None: list_of_graphs.append( Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes), - question=raw_dataset[index]["question"], - label=raw_dataset[index]["label"], - desc=raw_dataset[index]["desc"])) + question=data_i["question"], + label=data_i["label"], + desc=data_i["desc"])) print("data[" + str(index) + "] =", data) self.save(list_of_graphs, self.processed_paths[0]) From 8b3c3d699647bf9dff4211b74d400004d8379df1 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 14:33:52 -0800 Subject: [PATCH 040/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 123 ++++++++++++++++++-- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 74334aebbb0e..ca9dafe68ceb 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -6,9 +6,104 @@ from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer from typing import List - +import numpy as np +from pcst_fast import pcst_fast from torch_geometric.data import Data, InMemoryDataset +def retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk=3, topk_e=3, cost_e=0.5): + # from G-Retriever repo + c = 0.01 + if len(textual_nodes) == 0 or len(textual_edges) == 0: + desc = textual_nodes.to_csv(index=False) + '\n' + textual_edges.to_csv(index=False, columns=['src', 'edge_attr', 'dst']) + graph = Data(x=graph.x, edge_index=graph.edge_index, edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + return graph, desc + + root = -1 # unrooted + num_clusters = 1 + pruning = 'gw' + verbosity_level = 0 + if topk > 0: + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) + topk = min(topk, graph.num_nodes) + _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) + + n_prizes = torch.zeros_like(n_prizes) + n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() + else: + n_prizes = torch.zeros(graph.num_nodes) + + if topk_e > 0: + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + topk_e = min(topk_e, e_prizes.unique().size(0)) + + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + e_prizes[e_prizes < topk_e_values[-1]] = 0.0 + last_topk_e_value = topk_e + for k in range(topk_e): + indices = e_prizes == topk_e_values[k] + value = min((topk_e-k)/sum(indices), last_topk_e_value-c) + e_prizes[indices] = value + last_topk_e_value = value + # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) + else: + e_prizes = torch.zeros(graph.num_edges) + + costs = [] + edges = [] + vritual_n_prizes = [] + virtual_edges = [] + virtual_costs = [] + mapping_n = {} + mapping_e = {} + for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + prize_e = e_prizes[i] + if prize_e <= cost_e: + mapping_e[len(edges)] = i + edges.append((src, dst)) + costs.append(cost_e - prize_e) + else: + virtual_node_id = graph.num_nodes + len(vritual_n_prizes) + mapping_n[virtual_node_id] = i + virtual_edges.append((src, virtual_node_id)) + virtual_edges.append((virtual_node_id, dst)) + virtual_costs.append(0) + virtual_costs.append(0) + vritual_n_prizes.append(prize_e - cost_e) + + prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) + num_edges = len(edges) + if len(virtual_costs) > 0: + costs = np.array(costs+virtual_costs) + edges = np.array(edges+virtual_edges) + + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) + + selected_nodes = vertices[vertices < graph.num_nodes] + selected_edges = [mapping_e[e] for e in edges if e < num_edges] + virtual_vertices = vertices[vertices >= graph.num_nodes] + if len(virtual_vertices) > 0: + virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges+virtual_edges) + + edge_index = graph.edge_index[:, selected_edges] + selected_nodes = np.unique(np.concatenate([selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + + n = textual_nodes.iloc[selected_nodes] + e = textual_edges.iloc[selected_edges] + desc = n.to_csv(index=False)+'\n'+e.to_csv(index=False, columns=['src', 'edge_attr', 'dst']) + + mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + + x = graph.x[selected_nodes] + edge_attr = graph.edge_attr[selected_edges] + src = [mapping[i] for i in edge_index[0].tolist()] + dst = [mapping[i] for i in edge_index[1].tolist()] + edge_index = torch.LongTensor([src, dst]) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes)) + + return data, desc + class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids=None, attention_mask=None): @@ -151,9 +246,16 @@ def process(self) -> None: self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding + questions = [i['question'] for i in dataset] list_of_graphs = [] - #for index in tqdm(range(len(self.questions))): + # encode questions + print('Encoding questions...') + self.q_embs = self.text2embedding(self.model, self.tokenizer, + self.device, self.questions) + print("Encoding graphs...") + # (TODO) put TQDM back and remove prints once working for index, data_i in enumerate(self.raw_dataset): + print("data[" + str(index) + "] =", data_i) raw_nodes = {} raw_edges = [] for tri in data_i['graph']: @@ -185,11 +287,14 @@ def process(self) -> None: edges.edge_attr.tolist()) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) - list_of_graphs.append( - Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(nodes), - question=data_i["question"], - label=data_i["label"], - desc=data_i["desc"])) - print("data[" + str(index) + "] =", data) + question=f'Question: {data_i["question"]}\nAnswer: ' + label=('|').join(data_i['answer']).lower() + raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)) + psct_subgraph, desc = retrieval_via_pcst(raw_graph, self.q_embs[i], nodes, edges, topk=3, topk_e=5, cost_e=0.5) + pcst_subgraph["question"] = question + pcst_subgraph["label"] = label + list_of_graphs.append(pcst_subgraph) + print("raw_graph[" + str(index) + "] =", raw_graph) + print("pcst_subgraph[" + str(index) + "] =", pcst_subgraph) + print("") self.save(list_of_graphs, self.processed_paths[0]) From d01def1e0c96bf3ebc4fb984756af85afe2f2411 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 14:35:21 -0800 Subject: [PATCH 041/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ca9dafe68ceb..e1b091194062 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -246,7 +246,7 @@ def process(self) -> None: self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding - questions = [i['question'] for i in dataset] + questions = [i['question'] for i in self.raw_dataset] list_of_graphs = [] # encode questions print('Encoding questions...') From 0306c5ec1e0ec4d2d015f6482efaf36f791e0092 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 14:53:10 -0800 Subject: [PATCH 042/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e1b091194062..fa78dde2c10c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -246,11 +246,11 @@ def process(self) -> None: self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding - questions = [i['question'] for i in self.raw_dataset] + self.questions = [i['question'] for i in self.raw_dataset] list_of_graphs = [] # encode questions print('Encoding questions...') - self.q_embs = self.text2embedding(self.model, self.tokenizer, + q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print("Encoding graphs...") # (TODO) put TQDM back and remove prints once working From 92e7b134a89be621eb88af4d6dd2d8da1690c038 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 14:58:33 -0800 Subject: [PATCH 043/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index fa78dde2c10c..d0c84b2e62f8 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -255,7 +255,6 @@ def process(self) -> None: print("Encoding graphs...") # (TODO) put TQDM back and remove prints once working for index, data_i in enumerate(self.raw_dataset): - print("data[" + str(index) + "] =", data_i) raw_nodes = {} raw_edges = [] for tri in data_i['graph']: @@ -290,7 +289,7 @@ def process(self) -> None: question=f'Question: {data_i["question"]}\nAnswer: ' label=('|').join(data_i['answer']).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)) - psct_subgraph, desc = retrieval_via_pcst(raw_graph, self.q_embs[i], nodes, edges, topk=3, topk_e=5, cost_e=0.5) + psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[i], nodes, edges, topk=3, topk_e=5, cost_e=0.5) pcst_subgraph["question"] = question pcst_subgraph["label"] = label list_of_graphs.append(pcst_subgraph) From 77b7884c4133b820865dc012fe710ac05c26f64c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:08:05 -0800 Subject: [PATCH 044/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d0c84b2e62f8..39175cc49061 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -289,11 +289,11 @@ def process(self) -> None: question=f'Question: {data_i["question"]}\nAnswer: ' label=('|').join(data_i['answer']).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)) - psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[i], nodes, edges, topk=3, topk_e=5, cost_e=0.5) + psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) pcst_subgraph["question"] = question pcst_subgraph["label"] = label list_of_graphs.append(pcst_subgraph) print("raw_graph[" + str(index) + "] =", raw_graph) print("pcst_subgraph[" + str(index) + "] =", pcst_subgraph) - print("") + print("-*" * 10) self.save(list_of_graphs, self.processed_paths[0]) From 8df4944362e11308506e89ba49e9f94bd610387d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:11:21 -0800 Subject: [PATCH 045/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 39175cc49061..7dfee0504af6 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -290,10 +290,10 @@ def process(self) -> None: label=('|').join(data_i['answer']).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)) psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) - pcst_subgraph["question"] = question - pcst_subgraph["label"] = label - list_of_graphs.append(pcst_subgraph) + psct_subgraph["question"] = question + psct_subgraph["label"] = label + list_of_graphs.append(psct_subgraph) print("raw_graph[" + str(index) + "] =", raw_graph) - print("pcst_subgraph[" + str(index) + "] =", pcst_subgraph) + print("pcst_subgraph[" + str(index) + "] =", psct_subgraph) print("-*" * 10) self.save(list_of_graphs, self.processed_paths[0]) From e4edeacb6743b2183d21f2ded530bb9acc6c5944 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:12:44 -0800 Subject: [PATCH 046/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 7dfee0504af6..60627b03c390 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -288,11 +288,11 @@ def process(self) -> None: [edges.src.tolist(), edges.dst.tolist()]) question=f'Question: {data_i["question"]}\nAnswer: ' label=('|').join(data_i['answer']).lower() - raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)) + raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question psct_subgraph["label"] = label - list_of_graphs.append(psct_subgraph) + list_of_graphs.append(psct_subgraph.to("cpu")) print("raw_graph[" + str(index) + "] =", raw_graph) print("pcst_subgraph[" + str(index) + "] =", psct_subgraph) print("-*" * 10) From d2f3f8d82ff79a94c5734877f9e5456aa39a47a7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:14:49 -0800 Subject: [PATCH 047/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 60627b03c390..e4d7ee58fb3b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -292,6 +292,7 @@ def process(self) -> None: psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question psct_subgraph["label"] = label + psct_subgraph["desc"] = desc list_of_graphs.append(psct_subgraph.to("cpu")) print("raw_graph[" + str(index) + "] =", raw_graph) print("pcst_subgraph[" + str(index) + "] =", psct_subgraph) From ff988070bb0374720b2630dab58bf391d96ad922 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:17:37 -0800 Subject: [PATCH 048/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e4d7ee58fb3b..93ab7d24825c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -253,8 +253,7 @@ def process(self) -> None: q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print("Encoding graphs...") - # (TODO) put TQDM back and remove prints once working - for index, data_i in enumerate(self.raw_dataset): + for index, data_i in tqdm(enumerate(self.raw_dataset)): raw_nodes = {} raw_edges = [] for tri in data_i['graph']: @@ -294,7 +293,4 @@ def process(self) -> None: psct_subgraph["label"] = label psct_subgraph["desc"] = desc list_of_graphs.append(psct_subgraph.to("cpu")) - print("raw_graph[" + str(index) + "] =", raw_graph) - print("pcst_subgraph[" + str(index) + "] =", psct_subgraph) - print("-*" * 10) self.save(list_of_graphs, self.processed_paths[0]) From 54e42439aa4c851c95b30103964322b5cc186e04 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:20:41 -0800 Subject: [PATCH 049/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 93ab7d24825c..cceadbacfcf9 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -253,7 +253,8 @@ def process(self) -> None: q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print("Encoding graphs...") - for index, data_i in tqdm(enumerate(self.raw_dataset)): + for index in tqdm(len(self.raw_dataset)): + data_i = self.raw_dataset[i] raw_nodes = {} raw_edges = [] for tri in data_i['graph']: From 70479b40efa2be3fb35a206c8485397cc9fb964c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:23:23 -0800 Subject: [PATCH 050/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index cceadbacfcf9..a02d0af3c7b5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -253,7 +253,7 @@ def process(self) -> None: q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print("Encoding graphs...") - for index in tqdm(len(self.raw_dataset)): + for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[i] raw_nodes = {} raw_edges = [] From 8d4a54acdb038d6c19e9b9e21338e3ad75f27a6b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:25:31 -0800 Subject: [PATCH 051/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a02d0af3c7b5..e6667226884a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -254,7 +254,7 @@ def process(self) -> None: self.device, self.questions) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): - data_i = self.raw_dataset[i] + data_i = self.raw_dataset[index] raw_nodes = {} raw_edges = [] for tri in data_i['graph']: From 3fb96809747a63821178c4d38f2042b05b0e9693 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:39:43 -0800 Subject: [PATCH 052/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e6667226884a..1ee550610875 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -294,4 +294,6 @@ def process(self) -> None: psct_subgraph["label"] = label psct_subgraph["desc"] = desc list_of_graphs.append(psct_subgraph.to("cpu")) + if i>100: + break self.save(list_of_graphs, self.processed_paths[0]) From a26d364ee6f1b4fb8d52194223c5d60667dbdd63 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:43:00 -0800 Subject: [PATCH 053/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 1ee550610875..73fd95d405ca 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -294,6 +294,7 @@ def process(self) -> None: psct_subgraph["label"] = label psct_subgraph["desc"] = desc list_of_graphs.append(psct_subgraph.to("cpu")) - if i>100: + if index>100: + # just for initial setup, remove once bugfree break self.save(list_of_graphs, self.processed_paths[0]) From ddf124e50a81fc323d5fbcb62ee2f3f6be5aecb5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 4 Mar 2024 15:51:17 -0800 Subject: [PATCH 054/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 73fd95d405ca..e6667226884a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -294,7 +294,4 @@ def process(self) -> None: psct_subgraph["label"] = label psct_subgraph["desc"] = desc list_of_graphs.append(psct_subgraph.to("cpu")) - if index>100: - # just for initial setup, remove once bugfree - break self.save(list_of_graphs, self.processed_paths[0]) From 7f57d224f72ca8cb3c0760db2e33d852e28e2e64 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 10:09:21 -0800 Subject: [PATCH 055/752] WIP --- examples/LLM_plus_GNN.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 808979321a33..0c1de34fcb6a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -100,7 +100,7 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): - def __init__(self, **kwargs): + def __init__(self, graph_type, path, init_prompt, device): super().__init__() self.max_txt_len = 512 self.max_new_tokens = 32 @@ -116,7 +116,7 @@ def __init__(self, **kwargs): "device_map": "auto", "revision": "main", } - llm_model_path = kwargs["path"] + llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) self.tokenizer.pad_token_id = 0 @@ -145,7 +145,7 @@ def __init__(self, **kwargs): task_type="CAUSAL_LM", ) model = get_peft_model(model, config) - self.device = kwargs['device'] + self.device = device self.model = model.to(self.device) print('Finish loading LLAMA!') From 58dc3a6afa3f2add949c01e00421533007ede8b6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 12:22:16 -0800 Subject: [PATCH 056/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 62 ++++++++++----------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e6667226884a..edcf23823cc0 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -14,13 +14,13 @@ def retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk=3, topk_ # from G-Retriever repo c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + '\n' + textual_edges.to_csv(index=False, columns=['src', 'edge_attr', 'dst']) + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) graph = Data(x=graph.x, edge_index=graph.edge_index, edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc root = -1 # unrooted num_clusters = 1 - pruning = 'gw' + pruning = "gw" verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) @@ -91,7 +91,7 @@ def retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk=3, topk_ n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = n.to_csv(index=False)+'\n'+e.to_csv(index=False, columns=['src', 'edge_attr', 'dst']) + desc = n.to_csv(index=False)+"\n"+e.to_csv(index=False, columns=["src", "edge_attr", "dst"]) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -152,7 +152,7 @@ def forward(self, input_ids, att_mask): def sbert_text2embedding(model, tokenizer, device, text): try: encoding = tokenizer(text, padding=True, truncation=True, - return_tensors='pt') + return_tensors="pt") dataset = Dataset(input_ids=encoding.input_ids, attention_mask=encoding.attention_mask) @@ -203,16 +203,16 @@ class WebQSPDataset(InMemoryDataset): """ def __init__( self, - root: str = '', + root: str = "", force_reload: bool = False, ) -> None: - self.prompt = 'Please answer the given question.' + self.prompt = "Please answer the given question." self.graph = None - self.graph_type = 'Knowledge Graph' - self.model_name = 'sbert' + self.graph_type = "Knowledge Graph" + self.model_name = "sbert" self.device = torch.device( - 'cuda' if torch.cuda.is_available() else 'cpu') + "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) @@ -223,33 +223,33 @@ def raw_file_names(self) -> List[str]: @property def processed_file_names(self) -> List[str]: - return ['list_of_graphs.pt'] + return ["list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt"] def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( - [dataset['train'], dataset['validation'], dataset['test']]) + [dataset["train"], dataset["validation"], dataset["test"]]) self.split_idxs = { - 'train': - torch.arange(len(dataset['train'])), - 'val': - torch.arange(len(dataset['validation'])) + len(dataset['train']), - 'test': - torch.arange(len(dataset['test'])) + len(dataset['train']) + - len(dataset['validation']) + "train": + torch.arange(len(dataset["train"])), + "val": + torch.arange(len(dataset["validation"])) + len(dataset["train"]), + "test": + torch.arange(len(dataset["test"])) + len(dataset["train"]) + + len(dataset["validation"]) } def process(self) -> None: - pretrained_repo = 'sentence-transformers/all-roberta-large-v1' + pretrained_repo = "sentence-transformers/all-roberta-large-v1" self.model = Sentence_Transformer(pretrained_repo) self.model.to(self.device) self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) self.text2embedding = sbert_text2embedding - self.questions = [i['question'] for i in self.raw_dataset] + self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] # encode questions - print('Encoding questions...') + print("Encoding questions...") q_embs = self.text2embedding(self.model, self.tokenizer, self.device, self.questions) print("Encoding graphs...") @@ -257,7 +257,7 @@ def process(self) -> None: data_i = self.raw_dataset[index] raw_nodes = {} raw_edges = [] - for tri in data_i['graph']: + for tri in data_i["graph"]: h, r, t = tri h = h.lower() t = t.lower() @@ -266,16 +266,16 @@ def process(self) -> None: if t not in raw_nodes: raw_nodes[t] = len(raw_nodes) raw_edges.append({ - 'src': raw_nodes[h], - 'edge_attr': r, - 'dst': raw_nodes[t] + "src": raw_nodes[h], + "edge_attr": r, + "dst": raw_nodes[t] }) nodes = pd.DataFrame([{ - 'node_id': v, - 'node_attr': k - } for k, v in raw_nodes.items()], columns=['node_id', 'node_attr']) + "node_id": v, + "node_attr": k + } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"]) edges = pd.DataFrame(raw_edges, - columns=['src', 'edge_attr', 'dst']) + columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr.fillna("", inplace=True) x = self.text2embedding(self.model, self.tokenizer, self.device, @@ -286,8 +286,8 @@ def process(self) -> None: edges.edge_attr.tolist()) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) - question=f'Question: {data_i["question"]}\nAnswer: ' - label=('|').join(data_i['answer']).lower() + question=f"Question: {data_i["question"]}\nAnswer: " + label=("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question From 2b6e1ace7d8a65fcc40634e0ca740d4c9658f3fa Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 12:23:12 -0800 Subject: [PATCH 057/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index edcf23823cc0..a2093d7e981f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -286,7 +286,7 @@ def process(self) -> None: edges.edge_attr.tolist()) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) - question=f"Question: {data_i["question"]}\nAnswer: " + question=f"Question: {data_i['question']}\nAnswer: " label=("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) From 2f0526a49f34a11f5ef2475c895d662f3c3f0fc9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 14:41:18 -0800 Subject: [PATCH 058/752] WIP --- examples/LLM_plus_GNN.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0c1de34fcb6a..512ca55c4aab 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -145,8 +145,6 @@ def __init__(self, graph_type, path, init_prompt, device): task_type="CAUSAL_LM", ) model = get_peft_model(model, config) - self.device = device - self.model = model.to(self.device) print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( @@ -364,7 +362,7 @@ def main(): # Step 3: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, - init_prompt=dataset.prompt, device=dataset.device) + init_prompt=dataset.prompt) # Step 4 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] From 6be3ec87c26359b57c12975fd01c250925953602 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 19:43:59 -0800 Subject: [PATCH 059/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 512ca55c4aab..f3ab5a505748 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -100,7 +100,7 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): - def __init__(self, graph_type, path, init_prompt, device): + def __init__(self, graph_type, path, init_prompt): super().__init__() self.max_txt_len = 512 self.max_new_tokens = 32 From 2437ae9830b32f39ca2f7b9e70be2fe946d84671 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 19:48:16 -0800 Subject: [PATCH 060/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f3ab5a505748..8308c0f6f8b6 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -152,7 +152,7 @@ def __init__(self, graph_type, path, init_prompt): out_channels=1024, hidden_channels=1024, num_layers=4, - num_heads=4, + heads=4, ).to(self.model.device) self.projector = nn.Sequential( From b889c08cf38e857c0fb2a3927fd8d3bf51eed852 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 5 Mar 2024 19:51:04 -0800 Subject: [PATCH 061/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 8308c0f6f8b6..6666ea7dd8cc 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -128,7 +128,7 @@ def __init__(self, graph_type, path, init_prompt): **kwargs) print("Training LLAMA with LORA!") - model = prepare_model_for_int8_training(model) + self.model = prepare_model_for_int8_training(model) lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 From ddf324d24391c3dd43f6ce41581c70fcbace72fb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 7 Mar 2024 15:33:04 -0800 Subject: [PATCH 062/752] WIP --- examples/LLM_plus_GNN.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 6666ea7dd8cc..cbb01ac00da1 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -174,10 +174,11 @@ def maybe_autocast(self, dtype=torch.bfloat16): return contextlib.nullcontext() def encode_graphs(self, samples): - graphs = samples['graph'] - graphs = graphs.to(self.model.device) - n_embeds, _ = self.graph_encoder(graphs.x, graphs.edge_index.long(), - graphs.edge_attr) + x = samples.x.to(self.model.device) + edge_index = samples.edge_index.long().to(self.model.device) + edge_attr = samples.edge_attr.to(self.model.device) + n_embeds, _ = self.graph_encoder(x, edge_index.long(), + edge_attr) # mean pooling g_embeds = scatter(n_embeds, graphs.batch, dim=0, reduce='mean') From e7ce9b8a319f8b09166fc4ea3fa751d732e370c3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 7 Mar 2024 16:41:59 -0800 Subject: [PATCH 063/752] WIP --- examples/LLM_plus_GNN.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index cbb01ac00da1..e67d8239137a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -2,9 +2,6 @@ # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations # by 54% compared to the [LLM] baseline“. -# Original work uses LLAMA2 from Meta for LLM. -# This example uses GEMMA from google for the LLM -# as this is the current open source SOTA LLM. import contextlib import gc @@ -47,15 +44,6 @@ def adjust_learning_rate(param_group, LR, epoch): return lr -def collate_fn(original_batch): - batch = {} - for k in original_batch[0].keys(): - batch[k] = [d[k] for d in original_batch] - if 'graph' in batch: - batch['graph'] = Batch.from_data_list(batch['graph']) - return batch - - def compute_accuracy(eval_output): df = pd.concat([pd.DataFrame(d) for d in eval_output]) all_hit = [] @@ -351,14 +339,11 @@ def main(): test_dataset = [dataset[i] for i in idx_split['test']] train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, - pin_memory=True, shuffle=True, - collate_fn=collate_fn) + pin_memory=True, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=4, drop_last=False, - pin_memory=True, shuffle=False, - collate_fn=collate_fn) + pin_memory=True, shuffle=False) test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, - pin_memory=True, shuffle=False, - collate_fn=collate_fn) + pin_memory=True, shuffle=False) # Step 3: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" From f8464ed36289960b42130dd4afbaf6284d825eb8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 10:38:35 -0800 Subject: [PATCH 064/752] WIP --- examples/LLM_plus_GNN.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index e67d8239137a..a81ca01c1532 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -165,7 +165,7 @@ def encode_graphs(self, samples): x = samples.x.to(self.model.device) edge_index = samples.edge_index.long().to(self.model.device) edge_attr = samples.edge_attr.to(self.model.device) - n_embeds, _ = self.graph_encoder(x, edge_index.long(), + n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) # mean pooling @@ -174,7 +174,6 @@ def encode_graphs(self, samples): return g_embeds def forward(self, samples): - print("samples=", samples) # encode description, questions and labels questions = self.tokenizer(samples["question"], add_special_tokens=False) From 232d0920911985a752186b2ef7227203a2503d32 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:01:41 -0800 Subject: [PATCH 065/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index a81ca01c1532..f4af5dc844fb 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -169,7 +169,7 @@ def encode_graphs(self, samples): edge_attr) # mean pooling - g_embeds = scatter(n_embeds, graphs.batch, dim=0, reduce='mean') + g_embeds = scatter(n_embeds, samples.batch, dim=0, reduce='mean') return g_embeds From 0619c4c87968c99f24f2cdf233478d63752048a2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:04:33 -0800 Subject: [PATCH 066/752] WIP --- examples/LLM_plus_GNN.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f4af5dc844fb..196173071662 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -168,12 +168,14 @@ def encode_graphs(self, samples): n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) + batch = samples.batch.to(self.model.device) # mean pooling - g_embeds = scatter(n_embeds, samples.batch, dim=0, reduce='mean') + g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds def forward(self, samples): + print("samples=", samples) # encode description, questions and labels questions = self.tokenizer(samples["question"], add_special_tokens=False) From c75f23790e84642642c754a2750d652f602fe580 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:17:22 -0800 Subject: [PATCH 067/752] WIP --- examples/LLM_plus_GNN.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 196173071662..4ef59e2f9e02 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -333,7 +333,9 @@ def main(): dataset = WebQSPDataset() idx_split = dataset.split_idxs - + # Step 1: add IDs for each data point + for i in range(len(dataset)): + dataset[i]["id"] = i # Step 2: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] From 3e1d1cc44b326168ba40cb8037a8ceffd27372c9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:22:06 -0800 Subject: [PATCH 068/752] WIP --- examples/LLM_plus_GNN.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 4ef59e2f9e02..91d09e2aad5f 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -336,6 +336,7 @@ def main(): # Step 1: add IDs for each data point for i in range(len(dataset)): dataset[i]["id"] = i + print("dataset[i]=", dataset[i]) # Step 2: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] From 5da0636f603b330bc4669193039d140d88739b02 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:38:16 -0800 Subject: [PATCH 069/752] WIP --- examples/LLM_plus_GNN.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 91d09e2aad5f..1df059a831e5 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -336,7 +336,8 @@ def main(): # Step 1: add IDs for each data point for i in range(len(dataset)): dataset[i]["id"] = i - print("dataset[i]=", dataset[i]) + print("dataset[i]['id']=", dataset[i]['id']) + # Step 2: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] From c7ccf2319ce778de30eeb830e4fb7c3e533c2c9a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 11:39:22 -0800 Subject: [PATCH 070/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 1df059a831e5..28389fbe8f81 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -335,8 +335,8 @@ def main(): idx_split = dataset.split_idxs # Step 1: add IDs for each data point for i in range(len(dataset)): - dataset[i]["id"] = i - print("dataset[i]['id']=", dataset[i]['id']) + dataset[i].id = i + print("dataset[i].id=", dataset[i].id) # Step 2: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] From a0a54b2c773668c281e6fd720dfc7301fce79e6e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 12:51:52 -0800 Subject: [PATCH 071/752] WIP --- examples/LLM_plus_GNN.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 28389fbe8f81..df541073bec8 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -335,7 +335,9 @@ def main(): idx_split = dataset.split_idxs # Step 1: add IDs for each data point for i in range(len(dataset)): - dataset[i].id = i + data_pt = dataset[i] + data_pt.id = i + dataset[i] = data_pt print("dataset[i].id=", dataset[i].id) # Step 2: Build Node Classification Dataset From 2c20442e19f7403b667c88c8ae42b7b804451b1d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 12:59:48 -0800 Subject: [PATCH 072/752] WIP --- examples/LLM_plus_GNN.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index df541073bec8..f7a2ca3fd8c8 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -177,6 +177,7 @@ def encode_graphs(self, samples): def forward(self, samples): print("samples=", samples) # encode description, questions and labels + batch_size = len(samples["question"]) questions = self.tokenizer(samples["question"], add_special_tokens=False) descriptions = self.tokenizer(samples["desc"], @@ -196,7 +197,6 @@ def forward(self, samples): graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) - batch_size = len(samples['id']) batch_inputs_embeds = [] batch_attention_mask = [] batch_label_input_ids = [] @@ -250,6 +250,7 @@ def forward(self, samples): def inference(self, samples): # encode description and questions + batch_size = len(samples['question']) questions = self.tokenizer(samples["question"], add_special_tokens=False) descriptions = self.tokenizer(samples["desc"], @@ -267,7 +268,7 @@ def inference(self, samples): graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) - batch_size = len(samples['id']) + batch_inputs_embeds = [] batch_attention_mask = [] for i in range(batch_size): @@ -308,7 +309,6 @@ def inference(self, samples): pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) return { - 'id': samples['id'], 'pred': pred, 'label': samples['label'], 'question': samples['question'], @@ -333,14 +333,8 @@ def main(): dataset = WebQSPDataset() idx_split = dataset.split_idxs - # Step 1: add IDs for each data point - for i in range(len(dataset)): - data_pt = dataset[i] - data_pt.id = i - dataset[i] = data_pt - print("dataset[i].id=", dataset[i].id) - - # Step 2: Build Node Classification Dataset + + # Step 1: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] @@ -352,12 +346,12 @@ def main(): test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, pin_memory=True, shuffle=False) - # Step 3: Build Model + # Step 2: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, init_prompt=dataset.prompt) - # Step 4 Set Optimizer + # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] lr = 1e-5 optimizer = torch.optim.AdamW([ @@ -373,7 +367,7 @@ def main(): all params: {all_param} || \ trainable%: {100 * trainable_params / all_param}") - # Step 5. Training + # Step 4 Training num_training_steps = num_epochs * len(train_loader) progress_bar = tqdm(range(num_training_steps)) @@ -420,7 +414,7 @@ def main(): torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() - # Step 5. Evaluating + # Step 5 Evaluating print("Final Evaluation...") model.eval() eval_output = [] @@ -432,7 +426,7 @@ def main(): progress_bar_test.update(1) - # Step 6. Post-processing & compute metrics + # Step 6 Post-processing & compute metrics acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') From ee34e8f1e7563be5c1121aafefefa34f6a66ae2c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:02:34 -0800 Subject: [PATCH 073/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f7a2ca3fd8c8..33c906fef746 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -155,7 +155,7 @@ def maybe_autocast(self, dtype=torch.bfloat16): # if on cpu, don't use autocast # if on gpu, use autocast with dtype if provided, # otherwise use torch.float16 - enable_autocast = self.device != torch.device("cpu") + enable_autocast = self.model.device != torch.device("cpu") if enable_autocast: return torch.cuda.amp.autocast(dtype=dtype) else: From 3d7328a5859e89ac6033d0f7db13f8fe4a04c74c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:03:20 -0800 Subject: [PATCH 074/752] WIP --- examples/LLM_plus_GNN.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 33c906fef746..f2158a52791b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -175,7 +175,6 @@ def encode_graphs(self, samples): return g_embeds def forward(self, samples): - print("samples=", samples) # encode description, questions and labels batch_size = len(samples["question"]) questions = self.tokenizer(samples["question"], From 84c74803338bb062e93fedb21c04d88e1d598aa2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:17:58 -0800 Subject: [PATCH 075/752] WIP --- examples/LLM_plus_GNN.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f2158a52791b..17387a98ddd2 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -195,7 +195,6 @@ def forward(self, samples): # encode graphs graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) - batch_inputs_embeds = [] batch_attention_mask = [] batch_label_input_ids = [] @@ -267,7 +266,6 @@ def inference(self, samples): graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) - batch_inputs_embeds = [] batch_attention_mask = [] for i in range(batch_size): @@ -277,9 +275,13 @@ def inference(self, samples): i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) + try: + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) + except: + print("samples =", samples) + print("graph_embeds =", graph_embeds) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) From 2bc229c0cb221f796a858714c009d091e1621e8b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:24:01 -0800 Subject: [PATCH 076/752] WIP --- examples/LLM_plus_GNN.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 17387a98ddd2..b494527cfd2b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -207,9 +207,14 @@ def forward(self, samples): i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) + try: + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) + except: + print("samples =", samples) + print("graph_embeds =", graph_embeds) + quit() batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -275,13 +280,9 @@ def inference(self, samples): i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - try: - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) - except: - print("samples =", samples) - print("graph_embeds =", graph_embeds) + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) From e77e60af0eaf513f10db920c7e5f6c42f68005ce Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:35:27 -0800 Subject: [PATCH 077/752] WIP --- examples/LLM_plus_GNN.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index b494527cfd2b..3e2dab156e6d 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -214,6 +214,7 @@ def forward(self, samples): except: print("samples =", samples) print("graph_embeds =", graph_embeds) + print("len(graph_embeds) =", len(graph_embeds)) quit() batch_inputs_embeds.append(inputs_embeds) From 713bb245f8603b247b988aff897fd54e743a62a4 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:46:32 -0800 Subject: [PATCH 078/752] WIP --- examples/LLM_plus_GNN.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 3e2dab156e6d..79da576a1fec 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -167,11 +167,10 @@ def encode_graphs(self, samples): edge_attr = samples.edge_attr.to(self.model.device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - + print("len(n_embeds) =", len(n_embeds)) batch = samples.batch.to(self.model.device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') - return g_embeds def forward(self, samples): @@ -207,16 +206,9 @@ def forward(self, samples): i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - try: inputs_embeds = torch.cat( [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) - except: - print("samples =", samples) - print("graph_embeds =", graph_embeds) - print("len(graph_embeds) =", len(graph_embeds)) - quit() - batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 58be475e77f0f9f7b2312f07d3079633368a2319 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:48:06 -0800 Subject: [PATCH 079/752] WIP --- examples/LLM_plus_GNN.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 79da576a1fec..41cafdacd41c 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -206,9 +206,9 @@ def forward(self, samples): i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 767af60e3f1adbb8aea9c1646c2cce1cdb486367 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:56:04 -0800 Subject: [PATCH 080/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 41cafdacd41c..edba4389fedd 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -331,7 +331,7 @@ def main(): # Step 1: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] - val_dataset = [dataset[i] for i in idx_split['val']] + val_dataset = [dataset[i] for i in idx_split['val'] if dataset[i].x.numel() > 0] test_dataset = [dataset[i] for i in idx_split['test']] train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, From 688170a7f44339ded664e5646f134e08be2ba66b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 13:59:08 -0800 Subject: [PATCH 081/752] WIP --- examples/LLM_plus_GNN.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index edba4389fedd..efdbb36c5ff4 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -167,7 +167,6 @@ def encode_graphs(self, samples): edge_attr = samples.edge_attr.to(self.model.device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - print("len(n_embeds) =", len(n_embeds)) batch = samples.batch.to(self.model.device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') @@ -206,9 +205,15 @@ def forward(self, samples): i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) + try: + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) + except: + print("samples =", samples) + print("len(graph_embeds) =", len(graph_embeds)) + print("ptr =", ptr) + quit() batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX @@ -331,7 +336,7 @@ def main(): # Step 1: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] - val_dataset = [dataset[i] for i in idx_split['val'] if dataset[i].x.numel() > 0] + val_dataset = [dataset[i] for i in idx_split['val']] # if dataset[i].x.numel() > 0 test_dataset = [dataset[i] for i in idx_split['test']] train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, From e18b6bb76149a0e8d8b17c957e6eadf1777f4959 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 14:06:13 -0800 Subject: [PATCH 082/752] WIP --- examples/LLM_plus_GNN.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index efdbb36c5ff4..2db7065d3380 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -174,12 +174,12 @@ def encode_graphs(self, samples): def forward(self, samples): # encode description, questions and labels - batch_size = len(samples["question"]) - questions = self.tokenizer(samples["question"], + batch_size = len(samples.question) + questions = self.tokenizer(samples.question, add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], + descriptions = self.tokenizer(samples.desc, add_special_tokens=False) - labels = self.tokenizer(samples["label"], add_special_tokens=False) + labels = self.tokenizer(samples.label, add_special_tokens=False) # encode special tokens eos_tokens = self.tokenizer(EOS, add_special_tokens=False) @@ -212,7 +212,7 @@ def forward(self, samples): except: print("samples =", samples) print("len(graph_embeds) =", len(graph_embeds)) - print("ptr =", ptr) + print("samples.ptr =", samples.ptr) quit() batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -377,7 +377,8 @@ def main(): epoch_loss = 0. for step, batch in enumerate(train_loader): - + if step > 20: + break optimizer.zero_grad() loss = model(batch) loss.backward() From bfb2c9e240c766db0df79713f78e9d42a96d7e42 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 15:39:21 -0800 Subject: [PATCH 083/752] WIP --- examples/LLM_plus_GNN.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 2db7065d3380..0fa7dd06599b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -196,6 +196,7 @@ def forward(self, samples): batch_inputs_embeds = [] batch_attention_mask = [] batch_label_input_ids = [] + num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] for i in range(batch_size): # Add bos & eos token label_input_ids = labels.input_ids[ @@ -205,15 +206,11 @@ def forward(self, samples): i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.model.device)) - try: - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) - except: - print("samples =", samples) - print("len(graph_embeds) =", len(graph_embeds)) - print("samples.ptr =", samples.ptr) - quit() + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i]) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 273b5de05c450813539de1b860ab0914ca801faa Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 15:41:15 -0800 Subject: [PATCH 084/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0fa7dd06599b..1c82800ff88e 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -208,7 +208,7 @@ def forward(self, samples): torch.tensor(input_ids).to(self.model.device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i]) + to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) From 7413e0fc1b11084093cd1abaf1798ee6b539ef5f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 15:49:29 -0800 Subject: [PATCH 085/752] WIP --- examples/LLM_plus_GNN.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 1c82800ff88e..0c314e15e631 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -374,8 +374,6 @@ def main(): epoch_loss = 0. for step, batch in enumerate(train_loader): - if step > 20: - break optimizer.zero_grad() loss = model(batch) loss.backward() From 0a7451d782bedf98a763efa355e6a20d2b4691d2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 8 Mar 2024 16:19:56 -0800 Subject: [PATCH 086/752] WIP --- examples/LLM_plus_GNN.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0c314e15e631..8601b31ab531 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -392,7 +392,7 @@ def main(): progress_bar.update(1) - print(f"Epoch: {epoch}|{num_epochs}, \ + print(f"Epoch: {epoch + 1}|{num_epochs}, \ Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") val_loss = 0. @@ -403,9 +403,9 @@ def main(): loss = model(batch) val_loss += loss.item() val_loss = val_loss / len(val_loader) - print(f"Epoch: {epoch}|{num_epochs}: Val Loss: {val_loss}") + print(f"Epoch: {epoch + 1}|{num_epochs}: Val Loss: {val_loss}") - print(f'Epoch {epoch} Val Loss {val_loss}') + print(f'Epoch {epoch + 1} Val Loss {val_loss}') torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From 450331e7af5b5dc113a28c90afa0b2671c4e7a61 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Sun, 10 Mar 2024 10:07:47 -0700 Subject: [PATCH 087/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 8601b31ab531..bde7cefc1866 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,4 +1,4 @@ -# This example implements G-retriever +# This example implements G-retriever in PyG # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations # by 54% compared to the [LLM] baseline“. From 3cab62bdd9760d777c4f62ccba0c378e15da656a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Sun, 10 Mar 2024 10:08:14 -0700 Subject: [PATCH 088/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index bde7cefc1866..0e8fb68d6234 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,4 +1,4 @@ -# This example implements G-retriever in PyG +# This example implements G-retriever using PyG # https://github.com/XiaoxinHe/G-Retriever # “G-Retriever significantly reduces hallucinations # by 54% compared to the [LLM] baseline“. From 43703d8386d2cfc315cc3156d5d4a20455c3290e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 09:43:27 -0700 Subject: [PATCH 089/752] WIP --- examples/LLM_plus_GNN.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0e8fb68d6234..62c370eee153 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -7,6 +7,7 @@ import gc import math import re +import time import pandas as pd import torch @@ -325,7 +326,7 @@ def print_trainable_params(self): return trainable_params, all_param -def main(): +def main(since): seed_everything(42) dataset = WebQSPDataset() @@ -374,6 +375,9 @@ def main(): epoch_loss = 0. for step, batch in enumerate(train_loader): + if epoch == 0 and step == 0: + print("Training beginning...") + print("Total Prep Time (prep_time) =", prep_time) optimizer.zero_grad() loss = model(batch) loss.backward() @@ -425,10 +429,14 @@ def main(): # Step 6 Post-processing & compute metrics acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') - + return prep_time if __name__ == "__main__": - main() + since = time.time() + main(since) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() + e2e_time = round(time.time() - since, 2) + print("E2E time (e2e_time) =", e2e_time) + print("E2E time minus Prep Time =", e2e_time - prep_time) From 9cc4b803a2024ec114f8c1000128c87c91e61632 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 09:45:01 -0700 Subject: [PATCH 090/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 62c370eee153..6e51dfa4413a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -433,7 +433,7 @@ def main(since): if __name__ == "__main__": since = time.time() - main(since) + prep_time = main(since) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 7d5216bf012f5eac0dcb2ba6724a722990eb02f8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 09:47:57 -0700 Subject: [PATCH 091/752] WIP --- examples/LLM_plus_GNN.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 6e51dfa4413a..714e8344cc2c 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -377,6 +377,7 @@ def main(since): for step, batch in enumerate(train_loader): if epoch == 0 and step == 0: print("Training beginning...") + prep_time = round(time.time() - since, 2) print("Total Prep Time (prep_time) =", prep_time) optimizer.zero_grad() loss = model(batch) From 5fe6e0e227f6753e6b4ba1c3d9db16d9eda6c323 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:06:00 -0700 Subject: [PATCH 092/752] WIP --- examples/LLM_plus_GNN.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 714e8344cc2c..c2afd1d61155 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -96,12 +96,12 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { - "max_memory": { - 0: '20GiB', - 1: '20GiB', - 2: '20GiB', - 3: '20GiB' - }, + # "max_memory": { + # 0: '20GiB', + # 1: '20GiB', + # 2: '20GiB', + # 3: '20GiB' + # }, "device_map": "auto", "revision": "main", } From 835b3ceceb7a8a0647df3737048da4ccc7df5378 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:07:20 -0700 Subject: [PATCH 093/752] WIP --- examples/LLM_plus_GNN.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index c2afd1d61155..f5cc3ab57b23 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -96,12 +96,12 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { - # "max_memory": { - # 0: '20GiB', - # 1: '20GiB', - # 2: '20GiB', - # 3: '20GiB' - # }, + "max_memory": { + 0: '40GiB', + # 1: '20GiB', + # 2: '20GiB', + # 3: '20GiB' + }, "device_map": "auto", "revision": "main", } From 4a1de19478baf1987e1e08e8e98fd0c0798127db Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:10:48 -0700 Subject: [PATCH 094/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f5cc3ab57b23..7eeb7b99f899 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -97,7 +97,7 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { "max_memory": { - 0: '40GiB', + 0: '100GiB', # 1: '20GiB', # 2: '20GiB', # 3: '20GiB' From 3cd82b6c0ca282066bb82524fb4d3230b27542c4 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:13:09 -0700 Subject: [PATCH 095/752] WIP --- examples/LLM_plus_GNN.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 7eeb7b99f899..f67e9d01c599 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -96,13 +96,13 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { - "max_memory": { - 0: '100GiB', - # 1: '20GiB', - # 2: '20GiB', - # 3: '20GiB' - }, - "device_map": "auto", + # "max_memory": { + # 0: '20GiB', + # 1: '20GiB', + # 2: '20GiB', + # 3: '20GiB' + # }, + # "device_map": "auto", "revision": "main", } llm_model_path = path @@ -133,7 +133,7 @@ def __init__(self, graph_type, path, init_prompt): bias="none", task_type="CAUSAL_LM", ) - model = get_peft_model(model, config) + model = get_peft_model(model, config).to("cuda:0") print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( From 9d39ce20193c45bf8082bdc486a7f4831019040c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:16:31 -0700 Subject: [PATCH 096/752] WIP --- examples/LLM_plus_GNN.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f67e9d01c599..733eb7da2494 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -133,7 +133,9 @@ def __init__(self, graph_type, path, init_prompt): bias="none", task_type="CAUSAL_LM", ) - model = get_peft_model(model, config).to("cuda:0") + self.model = get_peft_model(self.model, config) + model.device = "cuda:0" + model.to(model.device) print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( From 982aea82af41cb238538095c91462e59ca391b7e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:17:22 -0700 Subject: [PATCH 097/752] WIP --- examples/LLM_plus_GNN.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 733eb7da2494..952907d3e013 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -134,8 +134,7 @@ def __init__(self, graph_type, path, init_prompt): task_type="CAUSAL_LM", ) self.model = get_peft_model(self.model, config) - model.device = "cuda:0" - model.to(model.device) + self.model.to("cuda:0") print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( From e4f1596ddf621aee5c97091990b08a7f211a3761 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:18:41 -0700 Subject: [PATCH 098/752] WIP --- examples/LLM_plus_GNN.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 952907d3e013..0538b86f4665 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -96,13 +96,13 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { - # "max_memory": { - # 0: '20GiB', - # 1: '20GiB', - # 2: '20GiB', - # 3: '20GiB' - # }, - # "device_map": "auto", + "max_memory": { + 0: '20GiB', + 1: '20GiB', + 2: '20GiB', + 3: '20GiB' + }, + "device_map": "auto", "revision": "main", } llm_model_path = path @@ -134,7 +134,6 @@ def __init__(self, graph_type, path, init_prompt): task_type="CAUSAL_LM", ) self.model = get_peft_model(self.model, config) - self.model.to("cuda:0") print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( From 5be25245595dce7347d2455b7b8d8b4cb4637674 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:21:08 -0700 Subject: [PATCH 099/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0538b86f4665..a2842cab19f4 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -99,8 +99,8 @@ def __init__(self, graph_type, path, init_prompt): "max_memory": { 0: '20GiB', 1: '20GiB', - 2: '20GiB', - 3: '20GiB' + # 2: '20GiB', + # 3: '20GiB' }, "device_map": "auto", "revision": "main", From fbd3293fc77cb58ec374acbbcba4495c68d6c72f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:26:57 -0700 Subject: [PATCH 100/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index a2842cab19f4..1657e0c07b16 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -97,8 +97,8 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') kwargs = { "max_memory": { - 0: '20GiB', - 1: '20GiB', + 0: '100GiB', + # 1: '20GiB', # 2: '20GiB', # 3: '20GiB' }, From b3bffa4eb6cc3ae5ea9c59b84f6b04b5f7ea8888 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 10:36:41 -0700 Subject: [PATCH 101/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 1657e0c07b16..463352b11528 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -102,7 +102,7 @@ def __init__(self, graph_type, path, init_prompt): # 2: '20GiB', # 3: '20GiB' }, - "device_map": "auto", + # "device_map": "auto", "revision": "main", } llm_model_path = path From 522386104c0438f777416f73900d01c554b6a269 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:02:00 -0700 Subject: [PATCH 102/752] WIP --- examples/LLM_plus_GNN.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 463352b11528..48bc82f2b12f 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,8 +1,11 @@ -# This example implements G-retriever using PyG -# https://github.com/XiaoxinHe/G-Retriever -# “G-Retriever significantly reduces hallucinations -# by 54% compared to the [LLM] baseline“. - +'''This example implements G-retriever using PyG. +Original Paper: https://arxiv.org/abs/2402.07630 +“G-Retriever significantly reduces hallucinations +by 54% compared to the [LLM] baseline“. + +requirements on top of basic PyG: +pip install peft datasets transformers pcst_fast sentencepiece +''' import contextlib import gc import math @@ -95,16 +98,22 @@ def __init__(self, graph_type, path, init_prompt): self.max_new_tokens = 32 print('Loading LLAMA') - kwargs = { - "max_memory": { - 0: '100GiB', - # 1: '20GiB', - # 2: '20GiB', - # 3: '20GiB' - }, - # "device_map": "auto", - "revision": "main", - } + avail_gpus = torch.cuda.is_available() + gpus_2_use_4_llm = min(avail_gpus, 4) + kwargs = {"revision": "main",} + max_mem_dict = {} + mem_total = 0 + for i in range(gpus_2_use_4_llm): + available_mem = torch.cuda.mem_get_info(0)[0] // 1024 ** 3 + mem_total += avilable_mem + max_mem_dict[i] = available_mem + assert mem_total >= 80, \ + "Need ~80GB of GPU RAM across all GPUs on device, only " \ + + str(mem_total) + "GB available across " + str(avail_gpus) \ + + " GPUs" + kwargs["max_memory"] = max_mem_dict + if gpus_2_use_4_llm == 4: + kwargs["device_map"] = "auto" llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) From c66513742dfad0d8368960141606eda536320c2a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:02:17 -0700 Subject: [PATCH 103/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 48bc82f2b12f..cb9117feb136 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -4,7 +4,7 @@ by 54% compared to the [LLM] baseline“. requirements on top of basic PyG: -pip install peft datasets transformers pcst_fast sentencepiece +pip install peft datasets transformers pcst_fast sentencepiece tqdm pandas ''' import contextlib import gc From 08e518cfe3ac741a95c4cb115bf2f963ea647b83 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:07:16 -0700 Subject: [PATCH 104/752] WIP --- examples/LLM_plus_GNN.py | 3 ++- torch_geometric/datasets/web_qsp_dataset.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index cb9117feb136..ab0c316bb963 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -104,7 +104,7 @@ def __init__(self, graph_type, path, init_prompt): max_mem_dict = {} mem_total = 0 for i in range(gpus_2_use_4_llm): - available_mem = torch.cuda.mem_get_info(0)[0] // 1024 ** 3 + available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += avilable_mem max_mem_dict[i] = available_mem assert mem_total >= 80, \ @@ -114,6 +114,7 @@ def __init__(self, graph_type, path, init_prompt): kwargs["max_memory"] = max_mem_dict if gpus_2_use_4_llm == 4: kwargs["device_map"] = "auto" + print("Setting up LLAMA w/ kwargs =", kwargs) llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a2093d7e981f..9db3d0fa00b5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -150,6 +150,10 @@ def forward(self, input_ids, att_mask): def sbert_text2embedding(model, tokenizer, device, text): + # print("type(model)=", type(model)) + # print("type(tokenizer)=", type(tokenizer)) + # print("type(device)=", type(device)) + # print("type(text)=", type(text)) try: encoding = tokenizer(text, padding=True, truncation=True, return_tensors="pt") From 84efaef8d17b4b656f375265dd0946f579d67c2e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:08:49 -0700 Subject: [PATCH 105/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index ab0c316bb963..2d51ac8a12dd 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -105,7 +105,7 @@ def __init__(self, graph_type, path, init_prompt): mem_total = 0 for i in range(gpus_2_use_4_llm): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) - mem_total += avilable_mem + mem_total += available_mem max_mem_dict[i] = available_mem assert mem_total >= 80, \ "Need ~80GB of GPU RAM across all GPUs on device, only " \ From f806b8b0cb2307e4fb2dbcadd68f25762ab11aa3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:17:35 -0700 Subject: [PATCH 106/752] WIP --- examples/LLM_plus_GNN.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 2d51ac8a12dd..59d719e6182a 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -98,7 +98,8 @@ def __init__(self, graph_type, path, init_prompt): self.max_new_tokens = 32 print('Loading LLAMA') - avail_gpus = torch.cuda.is_available() + assert torch.cuda.is_available(), "GPU needed!" + avail_gpus = torch.cuda.device_count() gpus_2_use_4_llm = min(avail_gpus, 4) kwargs = {"revision": "main",} max_mem_dict = {} @@ -106,7 +107,7 @@ def __init__(self, graph_type, path, init_prompt): for i in range(gpus_2_use_4_llm): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += available_mem - max_mem_dict[i] = available_mem + max_mem_dict[i] = str(available_mem) + "GiB" assert mem_total >= 80, \ "Need ~80GB of GPU RAM across all GPUs on device, only " \ + str(mem_total) + "GB available across " + str(avail_gpus) \ From 2f5b3efe4347b15866568f3e9042e8e7a06f2ec5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:20:28 -0700 Subject: [PATCH 107/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 59d719e6182a..3b647b6dcce6 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -113,8 +113,8 @@ def __init__(self, graph_type, path, init_prompt): + str(mem_total) + "GB available across " + str(avail_gpus) \ + " GPUs" kwargs["max_memory"] = max_mem_dict - if gpus_2_use_4_llm == 4: - kwargs["device_map"] = "auto" + #if gpus_2_use_4_llm == 4: + kwargs["device_map"] = "auto" print("Setting up LLAMA w/ kwargs =", kwargs) llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, From a2e2a925ee9d2bb8fb788f34deaf10108dfdbbfc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:22:26 -0700 Subject: [PATCH 108/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 3b647b6dcce6..7139ab973564 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -113,8 +113,8 @@ def __init__(self, graph_type, path, init_prompt): + str(mem_total) + "GB available across " + str(avail_gpus) \ + " GPUs" kwargs["max_memory"] = max_mem_dict - #if gpus_2_use_4_llm == 4: - kwargs["device_map"] = "auto" + if gpus_2_use_4_llm > 1: + kwargs["device_map"] = "auto" print("Setting up LLAMA w/ kwargs =", kwargs) llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, From 0031f8489bd52691a60cdfb4cb97a72f48b4b526 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:23:59 -0700 Subject: [PATCH 109/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 7139ab973564..13121d7a5677 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -154,7 +154,7 @@ def __init__(self, graph_type, path, init_prompt): num_layers=4, heads=4, ).to(self.model.device) - + print("self.model.device =", self.model.device) self.projector = nn.Sequential( nn.Linear(1024, 2048), nn.Sigmoid(), From 164dd59fc51d835c33663bbc4715d6025c63ad4c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:34:44 -0700 Subject: [PATCH 110/752] WIP --- examples/LLM_plus_GNN.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 13121d7a5677..2b6110ede3a3 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -100,18 +100,27 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') assert torch.cuda.is_available(), "GPU needed!" avail_gpus = torch.cuda.device_count() - gpus_2_use_4_llm = min(avail_gpus, 4) kwargs = {"revision": "main",} max_mem_dict = {} + avail_mem_dict = {} mem_total = 0 - for i in range(gpus_2_use_4_llm): + gpus_2_use_4_llm = 0 + for i in range(avail_gpus): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += available_mem - max_mem_dict[i] = str(available_mem) + "GiB" + avail_mem_dict[i] = available_mem + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + if mem_total >= 80: + break + assert mem_total >= 80, \ "Need ~80GB of GPU RAM across all GPUs on device, only " \ + str(mem_total) + "GB available across " + str(avail_gpus) \ - + " GPUs" + + " GPUs" + + for i in range(gpus_2_use_4_llm): + max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict if gpus_2_use_4_llm > 1: kwargs["device_map"] = "auto" From bf2d0a2a073b0775b53e31539e51cf7d6f40a4bd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:37:12 -0700 Subject: [PATCH 111/752] WIP --- examples/LLM_plus_GNN.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 2b6110ede3a3..791cd90b1faf 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -135,6 +135,8 @@ def __init__(self, graph_type, path, init_prompt): torch_dtype=torch.float16, low_cpu_mem_usage=True, **kwargs) + if gpus_2_use_4_llm == 1: + model = model.to('cuda:0') print("Training LLAMA with LORA!") self.model = prepare_model_for_int8_training(model) From ab1533a356a808ff04a4d3eaa1f9985eaf8d3ce4 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:37:59 -0700 Subject: [PATCH 112/752] WIP --- examples/LLM_plus_GNN.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 791cd90b1faf..40811f657e1b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -111,6 +111,7 @@ def __init__(self, graph_type, path, init_prompt): avail_mem_dict[i] = available_mem gpus_2_use_4_llm += 1 # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications if mem_total >= 80: break From 36a3bb083383d73f2d4231d090feb32720842a19 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:38:36 -0700 Subject: [PATCH 113/752] WIP --- examples/LLM_plus_GNN.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 40811f657e1b..9bdda8215d41 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -109,17 +109,17 @@ def __init__(self, graph_type, path, init_prompt): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += available_mem avail_mem_dict[i] = available_mem - gpus_2_use_4_llm += 1 - # We want to use the minimum number of GPUs that LLM can fit on - # this is to minimize the need for interGPU communications - if mem_total >= 80: - break - - assert mem_total >= 80, \ - "Need ~80GB of GPU RAM across all GPUs on device, only " \ - + str(mem_total) + "GB available across " + str(avail_gpus) \ - + " GPUs" - + # gpus_2_use_4_llm += 1 + # # We want to use the minimum number of GPUs that LLM can fit on + # # this is to minimize the need for interGPU communications + # if mem_total >= 80: + # break + + # assert mem_total >= 80, \ + # "Need ~80GB of GPU RAM across all GPUs on device, only " \ + # + str(mem_total) + "GB available across " + str(avail_gpus) \ + # + " GPUs" + gpus_2_use_4_llm = 4 for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict From 966f7bc2e64ac53dc8045fc75f4bb5aae0fe5092 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:39:35 -0700 Subject: [PATCH 114/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 9bdda8215d41..f541936d0f69 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -119,7 +119,7 @@ def __init__(self, graph_type, path, init_prompt): # "Need ~80GB of GPU RAM across all GPUs on device, only " \ # + str(mem_total) + "GB available across " + str(avail_gpus) \ # + " GPUs" - gpus_2_use_4_llm = 4 + gpus_2_use_4_llm = 2 for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict From 320c396d501fa440d81d9e01a596b3da6c0d2e3a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:47:39 -0700 Subject: [PATCH 115/752] WIP --- examples/LLM_plus_GNN.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index f541936d0f69..0ee732e5e615 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -109,17 +109,16 @@ def __init__(self, graph_type, path, init_prompt): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += available_mem avail_mem_dict[i] = available_mem - # gpus_2_use_4_llm += 1 - # # We want to use the minimum number of GPUs that LLM can fit on - # # this is to minimize the need for interGPU communications - # if mem_total >= 80: - # break - - # assert mem_total >= 80, \ - # "Need ~80GB of GPU RAM across all GPUs on device, only " \ - # + str(mem_total) + "GB available across " + str(avail_gpus) \ - # + " GPUs" - gpus_2_use_4_llm = 2 + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications + if mem_total >= 80: + break + + assert mem_total >= 80, \ + "Need ~80GB of GPU RAM across all GPUs on device, only " \ + + str(mem_total) + "GB available across " + str(avail_gpus) \ + + " GPUs" for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict @@ -174,6 +173,8 @@ def __init__(self, graph_type, path, init_prompt): ).to(self.model.device) self.word_embedding = self.model.model.get_input_embeddings() + if gpus_2_use_4_llm == 1: + self.word_embedding = self.word_embedding.to(self.model.device) def maybe_autocast(self, dtype=torch.bfloat16): # if on cpu, don't use autocast From 44209511a3bdb94547dc9e73ebd6b58e91c124c2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:53:00 -0700 Subject: [PATCH 116/752] WIP --- examples/LLM_plus_GNN.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0ee732e5e615..1ba3677b6e84 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -122,8 +122,7 @@ def __init__(self, graph_type, path, init_prompt): for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict - if gpus_2_use_4_llm > 1: - kwargs["device_map"] = "auto" + kwargs["device_map"] = "auto" print("Setting up LLAMA w/ kwargs =", kwargs) llm_model_path = path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, @@ -135,8 +134,6 @@ def __init__(self, graph_type, path, init_prompt): torch_dtype=torch.float16, low_cpu_mem_usage=True, **kwargs) - if gpus_2_use_4_llm == 1: - model = model.to('cuda:0') print("Training LLAMA with LORA!") self.model = prepare_model_for_int8_training(model) @@ -173,8 +170,6 @@ def __init__(self, graph_type, path, init_prompt): ).to(self.model.device) self.word_embedding = self.model.model.get_input_embeddings() - if gpus_2_use_4_llm == 1: - self.word_embedding = self.word_embedding.to(self.model.device) def maybe_autocast(self, dtype=torch.bfloat16): # if on cpu, don't use autocast From 7b3d562e43671d9c52c1ed6da168b7e3fedc1db8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 11:55:50 -0700 Subject: [PATCH 117/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 1ba3677b6e84..522dbc643ffb 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -132,7 +132,7 @@ def __init__(self, graph_type, path, init_prompt): model = AutoModelForCausalLM.from_pretrained(llm_model_path, torch_dtype=torch.float16, - low_cpu_mem_usage=True, + low_cpu_mem_usage=bool(gpus_2_use_4_llm > 1), **kwargs) print("Training LLAMA with LORA!") From f9d88f226e00b33f92f1b43ae255b6ba85cd25fd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:00:17 -0700 Subject: [PATCH 118/752] WIP --- examples/LLM_plus_GNN.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 522dbc643ffb..3208f223d9f0 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -132,7 +132,7 @@ def __init__(self, graph_type, path, init_prompt): model = AutoModelForCausalLM.from_pretrained(llm_model_path, torch_dtype=torch.float16, - low_cpu_mem_usage=bool(gpus_2_use_4_llm > 1), + low_cpu_mem_usage=True, **kwargs) print("Training LLAMA with LORA!") @@ -206,9 +206,9 @@ def forward(self, samples): eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) bos_embeds = self.word_embedding( self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0]) + return_tensors='pt').input_ids[0].to(self.model.device)) pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + torch.tensor(self.tokenizer.pad_token_id).to(self.model.device)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) From 9b43a20a3c08ac46191463874fba5cf8e7cc8147 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:02:51 -0700 Subject: [PATCH 119/752] WIP --- examples/LLM_plus_GNN.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 3208f223d9f0..6bb208d1e545 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -109,16 +109,17 @@ def __init__(self, graph_type, path, init_prompt): available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) mem_total += available_mem avail_mem_dict[i] = available_mem - gpus_2_use_4_llm += 1 - # We want to use the minimum number of GPUs that LLM can fit on - # this is to minimize the need for interGPU communications - if mem_total >= 80: - break - - assert mem_total >= 80, \ - "Need ~80GB of GPU RAM across all GPUs on device, only " \ - + str(mem_total) + "GB available across " + str(avail_gpus) \ - + " GPUs" + # gpus_2_use_4_llm += 1 + # # We want to use the minimum number of GPUs that LLM can fit on + # # this is to minimize the need for interGPU communications + # if mem_total >= 80: + # break + + # assert mem_total >= 80, \ + # "Need ~80GB of GPU RAM across all GPUs on device, only " \ + # + str(mem_total) + "GB available across " + str(avail_gpus) \ + # + " GPUs" + gpus_2_use_4_llm = 4 for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict From ebeaa57974cd755d8034962dbae72aa7985fb94c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:05:10 -0700 Subject: [PATCH 120/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 6bb208d1e545..470311ab2bd0 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -119,7 +119,7 @@ def __init__(self, graph_type, path, init_prompt): # "Need ~80GB of GPU RAM across all GPUs on device, only " \ # + str(mem_total) + "GB available across " + str(avail_gpus) \ # + " GPUs" - gpus_2_use_4_llm = 4 + gpus_2_use_4_llm = 2 for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict From ff6039ed1af635c5f9fef2feb567fc03d7c520d5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:23:32 -0700 Subject: [PATCH 121/752] WIP --- examples/LLM_plus_GNN.py | 26 ++++++++++++--------- torch_geometric/datasets/web_qsp_dataset.py | 11 +++++---- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 470311ab2bd0..9321e5a8f65c 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -105,21 +105,25 @@ def __init__(self, graph_type, path, init_prompt): avail_mem_dict = {} mem_total = 0 gpus_2_use_4_llm = 0 + print("avail_gpus =", avail_gpus) for i in range(avail_gpus): + print("i =", i) available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) + print("avilable_mem_i =", available_mem) mem_total += available_mem avail_mem_dict[i] = available_mem - # gpus_2_use_4_llm += 1 - # # We want to use the minimum number of GPUs that LLM can fit on - # # this is to minimize the need for interGPU communications - # if mem_total >= 80: - # break - - # assert mem_total >= 80, \ - # "Need ~80GB of GPU RAM across all GPUs on device, only " \ - # + str(mem_total) + "GB available across " + str(avail_gpus) \ - # + " GPUs" - gpus_2_use_4_llm = 2 + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications + if mem_total >= 80: + break + + assert mem_total >= 80, \ + "Need ~80GB of GPU RAM across all GPUs on device, only " \ + + str(mem_total) + "GB available across " + str(avail_gpus) \ + + " GPUs" + + print("gpus_2_use_4_llm=",gpus_2_use_4_llm) for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 9db3d0fa00b5..b09146bab731 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -127,7 +127,7 @@ def __getitem__(self, index): class Sentence_Transformer(torch.nn.Module): - def __init__(self, pretrained_repo): + def __init__(self, pretrained_repo: str) -> None: super(Sentence_Transformer, self).__init__() print(f"inherit model weights from {pretrained_repo}") self.bert_model = AutoModel.from_pretrained(pretrained_repo) @@ -142,6 +142,7 @@ def mean_pooling(self, model_output, attention_mask): 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def forward(self, input_ids, att_mask): + bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) sentence_embeddings = self.mean_pooling(bert_out, att_mask) @@ -150,10 +151,10 @@ def forward(self, input_ids, att_mask): def sbert_text2embedding(model, tokenizer, device, text): - # print("type(model)=", type(model)) - # print("type(tokenizer)=", type(tokenizer)) - # print("type(device)=", type(device)) - # print("type(text)=", type(text)) + print("type(model)=", type(model)) + print("type(tokenizer)=", type(tokenizer)) + print("type(device)=", type(device)) + print("type(text)=", type(text)) try: encoding = tokenizer(text, padding=True, truncation=True, return_tensors="pt") From d548b37c9415636ced75426ad83f98fef4d73891 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:55:07 -0700 Subject: [PATCH 122/752] WIP --- examples/LLM_plus_GNN.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 9321e5a8f65c..7a965456a6c0 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -105,11 +105,8 @@ def __init__(self, graph_type, path, init_prompt): avail_mem_dict = {} mem_total = 0 gpus_2_use_4_llm = 0 - print("avail_gpus =", avail_gpus) for i in range(avail_gpus): - print("i =", i) available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) - print("avilable_mem_i =", available_mem) mem_total += available_mem avail_mem_dict[i] = available_mem gpus_2_use_4_llm += 1 @@ -123,7 +120,6 @@ def __init__(self, graph_type, path, init_prompt): + str(mem_total) + "GB available across " + str(avail_gpus) \ + " GPUs" - print("gpus_2_use_4_llm=",gpus_2_use_4_llm) for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict @@ -283,9 +279,9 @@ def inference(self, samples): eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) bos_embeds = self.word_embedding( self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0]) + return_tensors='pt').input_ids[0].to(self.model.device)) pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id)).unsqueeze(0) + torch.tensor(self.tokenizer.pad_token_id).to(self.model.device)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -390,15 +386,12 @@ def main(since): trainable%: {100 * trainable_params / all_param}") # Step 4 Training - num_training_steps = num_epochs * len(train_loader) - progress_bar = tqdm(range(num_training_steps)) - for epoch in range(num_epochs): - model.train() epoch_loss = 0. - - for step, batch in enumerate(train_loader): + loader = tqdm(enumerate(train_loader), + description="Epoch " + str(epoch)) + for step, batch in loader: if epoch == 0 and step == 0: print("Training beginning...") prep_time = round(time.time() - since, 2) @@ -419,8 +412,6 @@ def main(since): if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] - progress_bar.update(1) - print(f"Epoch: {epoch + 1}|{num_epochs}, \ Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") From dde77cca515df91e46911c333fb8fea97a40b8e5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 12:56:47 -0700 Subject: [PATCH 123/752] WIP --- examples/LLM_plus_GNN.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 7a965456a6c0..afcc97ca0433 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -389,8 +389,7 @@ def main(since): for epoch in range(num_epochs): model.train() epoch_loss = 0. - loader = tqdm(enumerate(train_loader), - description="Epoch " + str(epoch)) + loader = tqdm(enumerate(train_loader), desc="Epoch " + str(epoch)) for step, batch in loader: if epoch == 0 and step == 0: print("Training beginning...") From 27a410fa4d17dea9ef8520a89593f73a5355cd4c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:08:27 -0700 Subject: [PATCH 124/752] WIP --- examples/LLM_plus_GNN.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index afcc97ca0433..88760a4225ee 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -389,12 +389,12 @@ def main(since): for epoch in range(num_epochs): model.train() epoch_loss = 0. + if epoch == 0: + prep_time = round(time.time() - since, 2) + print("Total Prep Time (prep_time) =", prep_time) + print("Training beginning...") loader = tqdm(enumerate(train_loader), desc="Epoch " + str(epoch)) for step, batch in loader: - if epoch == 0 and step == 0: - print("Training beginning...") - prep_time = round(time.time() - since, 2) - print("Total Prep Time (prep_time) =", prep_time) optimizer.zero_grad() loss = model(batch) loss.backward() From ed6d86df1e7cc272ab70d0e3de3106675c407e91 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:10:17 -0700 Subject: [PATCH 125/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 88760a4225ee..9186862b6eeb 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -393,8 +393,8 @@ def main(since): prep_time = round(time.time() - since, 2) print("Total Prep Time (prep_time) =", prep_time) print("Training beginning...") - loader = tqdm(enumerate(train_loader), desc="Epoch " + str(epoch)) - for step, batch in loader: + loader = tqdm(train_loader, desc="Epoch " + str(epoch)) + for step, batch in enumerate(loader): optimizer.zero_grad() loss = model(batch) loss.backward() From cd96e4170110b0aeb02888a37c7ddb0118dda05b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:17:40 -0700 Subject: [PATCH 126/752] WIP --- examples/LLM_plus_GNN.py | 1 - torch_geometric/datasets/web_qsp_dataset.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 9186862b6eeb..a59d0650001b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -163,7 +163,6 @@ def __init__(self, graph_type, path, init_prompt): num_layers=4, heads=4, ).to(self.model.device) - print("self.model.device =", self.model.device) self.projector = nn.Sequential( nn.Linear(1024, 2048), nn.Sigmoid(), diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index b09146bab731..b6adec507fee 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -5,13 +5,16 @@ from tqdm import tqdm from torch.utils.data import DataLoader from transformers import AutoModel, AutoTokenizer -from typing import List +from typing import List, Tuple import numpy as np from pcst_fast import pcst_fast from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk=3, topk_e=3, cost_e=0.5): +def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes, textual_edges, topk=3, topk_e=3, cost_e=0.5) -> Tuple[Data, str]: # from G-Retriever repo + print("type(textual_nodes) =", type(textual_nodes)) + print("type(textual_edges) =", type(textual_edges)) + c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) @@ -133,8 +136,11 @@ def __init__(self, pretrained_repo: str) -> None: self.bert_model = AutoModel.from_pretrained(pretrained_repo) def mean_pooling(self, model_output, attention_mask): + print("type(model_output)=", type(model_output)) + print("type(attention_mask)=", type(attention_mask)) # First element of model_output contains all token embeddings token_embeddings = model_output[0] + print("type(token_embeddings)=", type(token_embeddings)) data_type = token_embeddings.dtype input_mask_expanded = attention_mask.unsqueeze(-1).expand( token_embeddings.size()).to(data_type) @@ -142,6 +148,8 @@ def mean_pooling(self, model_output, attention_mask): 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def forward(self, input_ids, att_mask): + print("type(input_ids)=", type(input_ids)) + print("type(att_mask)=", type(att_mask)) bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) From 1ec89d23f300bb7f88b5256ab777915f593af13f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:29:13 -0700 Subject: [PATCH 127/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 28 ++++++++------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index b6adec507fee..079006c0947f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -10,7 +10,7 @@ from pcst_fast import pcst_fast from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes, textual_edges, topk=3, topk_e=3, cost_e=0.5) -> Tuple[Data, str]: +def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, textual_edges: pd.DataFrame, topk: int=3, topk_e: int=3, cost_e: float=0.5) -> Tuple[Data, str]: # from G-Retriever repo print("type(textual_nodes) =", type(textual_nodes)) print("type(textual_edges) =", type(textual_edges)) @@ -135,34 +135,24 @@ def __init__(self, pretrained_repo: str) -> None: print(f"inherit model weights from {pretrained_repo}") self.bert_model = AutoModel.from_pretrained(pretrained_repo) - def mean_pooling(self, model_output, attention_mask): - print("type(model_output)=", type(model_output)) - print("type(attention_mask)=", type(attention_mask)) - # First element of model_output contains all token embeddings - token_embeddings = model_output[0] - print("type(token_embeddings)=", type(token_embeddings)) + def mean_pooling(self, token_embeddings: torch.Tensor, attention_mask: torch.Tensor): data_type = token_embeddings.dtype input_mask_expanded = attention_mask.unsqueeze(-1).expand( token_embeddings.size()).to(data_type) return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) - def forward(self, input_ids, att_mask): - print("type(input_ids)=", type(input_ids)) - print("type(att_mask)=", type(att_mask)) + def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor): + bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) - bert_out = self.bert_model(input_ids=input_ids, - attention_mask=att_mask) - sentence_embeddings = self.mean_pooling(bert_out, att_mask) + # First element of model_output contains all token embeddings + token_embeddings = bert_out[0] + sentence_embeddings = self.mean_pooling(token_embeddings, att_mask) sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) return sentence_embeddings -def sbert_text2embedding(model, tokenizer, device, text): - print("type(model)=", type(model)) - print("type(tokenizer)=", type(tokenizer)) - print("type(device)=", type(device)) - print("type(text)=", type(text)) +def sbert_text2embedding(model: Sentence_Transformer, tokenizer: torch.nn.Module, device: torch.device, text: List[str]) -> List[torch.Tensor]: try: encoding = tokenizer(text, padding=True, truncation=True, return_tensors="pt") @@ -267,6 +257,8 @@ def process(self) -> None: self.device, self.questions) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): + if index == 1: + quit() data_i = self.raw_dataset[index] raw_nodes = {} raw_edges = [] From 7255c2bd9a7529b97d75a9bc124344eb415c0e11 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:31:53 -0700 Subject: [PATCH 128/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 079006c0947f..420f09613fbe 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -12,9 +12,6 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, textual_edges: pd.DataFrame, topk: int=3, topk_e: int=3, cost_e: float=0.5) -> Tuple[Data, str]: # from G-Retriever repo - print("type(textual_nodes) =", type(textual_nodes)) - print("type(textual_edges) =", type(textual_edges)) - c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) @@ -110,6 +107,8 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids=None, attention_mask=None): + print("type(input_ids)=", type(input_ids)) + print("type(attention_mask)=", type(attention_mask)) super().__init__() self.data = { "input_ids": input_ids, @@ -120,6 +119,7 @@ def __len__(self): return self.data["input_ids"].size(0) def __getitem__(self, index): + print("type(index)=", index) if isinstance(index, torch.Tensor): index = index.item() batch_data = dict() From cde783aaa234bfa26a44a340a7f955faa1c008f7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:34:06 -0700 Subject: [PATCH 129/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 420f09613fbe..0f47455817be 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -106,7 +106,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids=None, attention_mask=None): + def __init__(self, input_ids: torch.Tensor=None, attention_mask: torch.Tensor=None): print("type(input_ids)=", type(input_ids)) print("type(attention_mask)=", type(attention_mask)) super().__init__() @@ -118,7 +118,7 @@ def __init__(self, input_ids=None, attention_mask=None): def __len__(self): return self.data["input_ids"].size(0) - def __getitem__(self, index): + def __getitem__(self, index: int): print("type(index)=", index) if isinstance(index, torch.Tensor): index = index.item() From 3dbe902b71cbad642e0e6142fa01b9868709e2da Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:34:19 -0700 Subject: [PATCH 130/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0f47455817be..8c05afc81e93 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -107,8 +107,6 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids: torch.Tensor=None, attention_mask: torch.Tensor=None): - print("type(input_ids)=", type(input_ids)) - print("type(attention_mask)=", type(attention_mask)) super().__init__() self.data = { "input_ids": input_ids, From dc300e35273c2ddd08e6c4a071aef7e9e4b04e8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:37:01 +0000 Subject: [PATCH 131/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 34 +++++---- torch_geometric/datasets/web_qsp_dataset.py | 78 +++++++++++++-------- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index a59d0650001b..9bd3fe95a2f8 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -17,13 +17,12 @@ import torch.nn as nn from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training from torch.nn.utils import clip_grad_norm_ -from torch_geometric.data import DataLoader from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer import torch_geometric from torch_geometric import seed_everything -from torch_geometric.data import Batch +from torch_geometric.data import Batch, DataLoader from torch_geometric.datasets import WebQSPDataset from torch_geometric.utils import scatter @@ -100,13 +99,15 @@ def __init__(self, graph_type, path, init_prompt): print('Loading LLAMA') assert torch.cuda.is_available(), "GPU needed!" avail_gpus = torch.cuda.device_count() - kwargs = {"revision": "main",} + kwargs = { + "revision": "main", + } max_mem_dict = {} avail_mem_dict = {} mem_total = 0 gpus_2_use_4_llm = 0 for i in range(avail_gpus): - available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024 ** 3) + available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024**3) mem_total += available_mem avail_mem_dict[i] = available_mem gpus_2_use_4_llm += 1 @@ -185,8 +186,7 @@ def encode_graphs(self, samples): x = samples.x.to(self.model.device) edge_index = samples.edge_index.long().to(self.model.device) edge_attr = samples.edge_attr.to(self.model.device) - n_embeds = self.graph_encoder(x, edge_index.long(), - edge_attr) + n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) batch = samples.batch.to(self.model.device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') @@ -195,10 +195,8 @@ def encode_graphs(self, samples): def forward(self, samples): # encode description, questions and labels batch_size = len(samples.question) - questions = self.tokenizer(samples.question, - add_special_tokens=False) - descriptions = self.tokenizer(samples.desc, - add_special_tokens=False) + questions = self.tokenizer(samples.question, add_special_tokens=False) + descriptions = self.tokenizer(samples.desc, add_special_tokens=False) labels = self.tokenizer(samples.label, add_special_tokens=False) # encode special tokens @@ -206,9 +204,11 @@ def forward(self, samples): eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) bos_embeds = self.word_embedding( self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to(self.model.device)) + return_tensors='pt').input_ids[0].to( + self.model.device)) pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to(self.model.device)).unsqueeze(0) + torch.tensor(self.tokenizer.pad_token_id).to( + self.model.device)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -278,9 +278,11 @@ def inference(self, samples): eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) bos_embeds = self.word_embedding( self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to(self.model.device)) + return_tensors='pt').input_ids[0].to( + self.model.device)) pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to(self.model.device)).unsqueeze(0) + torch.tensor(self.tokenizer.pad_token_id).to( + self.model.device)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -353,7 +355,8 @@ def main(since): # Step 1: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] - val_dataset = [dataset[i] for i in idx_split['val']] # if dataset[i].x.numel() > 0 + val_dataset = [dataset[i] + for i in idx_split['val']] # if dataset[i].x.numel() > 0 test_dataset = [dataset[i] for i in idx_split['test']] train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, @@ -445,6 +448,7 @@ def main(since): print(f'Test Acc {acc}') return prep_time + if __name__ == "__main__": since = time.time() prep_time = main(since) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 8c05afc81e93..bb096f5410d8 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,21 +1,30 @@ +from typing import List, Tuple + import datasets +import numpy as np import pandas as pd import torch import torch.nn.functional as F -from tqdm import tqdm +from pcst_fast import pcst_fast from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModel, AutoTokenizer -from typing import List, Tuple -import numpy as np -from pcst_fast import pcst_fast + from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, textual_edges: pd.DataFrame, topk: int=3, topk_e: int=3, cost_e: float=0.5) -> Tuple[Data, str]: + +def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, + textual_nodes: pd.DataFrame, + textual_edges: pd.DataFrame, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: # from G-Retriever repo c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) - graph = Data(x=graph.x, edge_index=graph.edge_index, edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + graph = Data(x=graph.x, edge_index=graph.edge_index, + edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc root = -1 # unrooted @@ -41,7 +50,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF last_topk_e_value = topk_e for k in range(topk_e): indices = e_prizes == topk_e_values[k] - value = min((topk_e-k)/sum(indices), last_topk_e_value-c) + value = min((topk_e - k) / sum(indices), last_topk_e_value - c) e_prizes[indices] = value last_topk_e_value = value # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) @@ -73,10 +82,11 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) num_edges = len(edges) if len(virtual_costs) > 0: - costs = np.array(costs+virtual_costs) - edges = np.array(edges+virtual_edges) + costs = np.array(costs + virtual_costs) + edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] @@ -84,14 +94,17 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF if len(virtual_vertices) > 0: virtual_vertices = vertices[vertices >= graph.num_nodes] virtual_edges = [mapping_n[i] for i in virtual_vertices] - selected_edges = np.array(selected_edges+virtual_edges) + selected_edges = np.array(selected_edges + virtual_edges) edge_index = graph.edge_index[:, selected_edges] - selected_nodes = np.unique(np.concatenate([selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + selected_nodes = np.unique( + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = n.to_csv(index=False)+"\n"+e.to_csv(index=False, columns=["src", "edge_attr", "dst"]) + desc = n.to_csv(index=False) + "\n" + e.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -100,13 +113,15 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataF src = [mapping[i] for i in edge_index[0].tolist()] dst = [mapping[i] for i in edge_index[1].tolist()] edge_index = torch.LongTensor([src, dst]) - data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes)) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) return data, desc class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids: torch.Tensor=None, attention_mask: torch.Tensor=None): + def __init__(self, input_ids: torch.Tensor = None, + attention_mask: torch.Tensor = None): super().__init__() self.data = { "input_ids": input_ids, @@ -133,7 +148,8 @@ def __init__(self, pretrained_repo: str) -> None: print(f"inherit model weights from {pretrained_repo}") self.bert_model = AutoModel.from_pretrained(pretrained_repo) - def mean_pooling(self, token_embeddings: torch.Tensor, attention_mask: torch.Tensor): + def mean_pooling(self, token_embeddings: torch.Tensor, + attention_mask: torch.Tensor): data_type = token_embeddings.dtype input_mask_expanded = attention_mask.unsqueeze(-1).expand( token_embeddings.size()).to(data_type) @@ -141,7 +157,8 @@ def mean_pooling(self, token_embeddings: torch.Tensor, attention_mask: torch.Ten 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor): - bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) + bert_out = self.bert_model(input_ids=input_ids, + attention_mask=att_mask) # First element of model_output contains all token embeddings token_embeddings = bert_out[0] @@ -150,7 +167,9 @@ def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor): return sentence_embeddings -def sbert_text2embedding(model: Sentence_Transformer, tokenizer: torch.nn.Module, device: torch.device, text: List[str]) -> List[torch.Tensor]: +def sbert_text2embedding(model: Sentence_Transformer, + tokenizer: torch.nn.Module, device: torch.device, + text: List[str]) -> List[torch.Tensor]: try: encoding = tokenizer(text, padding=True, truncation=True, return_tensors="pt") @@ -180,7 +199,8 @@ def sbert_text2embedding(model: Sentence_Transformer, tokenizer: torch.nn.Module # Concatenate the embeddings from all batches all_embeddings = torch.cat(all_embeddings, dim=0).cpu() except: # noqa - print("SBERT text embedding failed, returning torch.zeros((0, 1024))...") + print( + "SBERT text embedding failed, returning torch.zeros((0, 1024))...") return torch.zeros((0, 1024)) return all_embeddings @@ -217,14 +237,13 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - @property def raw_file_names(self) -> List[str]: return [] @property def processed_file_names(self) -> List[str]: - return ["list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt"] + return ["list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt"] def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") @@ -251,8 +270,8 @@ def process(self) -> None: list_of_graphs = [] # encode questions print("Encoding questions...") - q_embs = self.text2embedding(self.model, self.tokenizer, - self.device, self.questions) + q_embs = self.text2embedding(self.model, self.tokenizer, self.device, + self.questions) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): if index == 1: @@ -289,10 +308,13 @@ def process(self) -> None: edges.edge_attr.tolist()) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) - question=f"Question: {data_i['question']}\nAnswer: " - label=("|").join(data_i["answer"]).lower() - raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) + question = f"Question: {data_i['question']}\nAnswer: " + label = ("|").join(data_i["answer"]).lower() + raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(nodes)).to("cpu") + psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + nodes, edges, topk=3, + topk_e=5, cost_e=0.5) psct_subgraph["question"] = question psct_subgraph["label"] = label psct_subgraph["desc"] = desc From 633f11e6f7331b83fee72c23222cb0ac3988da67 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:37:49 -0700 Subject: [PATCH 132/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index a59d0650001b..59953e5a1e0b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -392,7 +392,7 @@ def main(since): prep_time = round(time.time() - since, 2) print("Total Prep Time (prep_time) =", prep_time) print("Training beginning...") - loader = tqdm(train_loader, desc="Epoch " + str(epoch)) + loader = tqdm(train_loader, desc="Epoch " + str(epoch + 1)) for step, batch in enumerate(loader): optimizer.zero_grad() loss = model(batch) From 6e196a9aed50831c971c6625b48f99b9ae28ed7b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:38:50 -0700 Subject: [PATCH 133/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index bb096f5410d8..3232fb4dbe31 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -132,7 +132,6 @@ def __len__(self): return self.data["input_ids"].size(0) def __getitem__(self, index: int): - print("type(index)=", index) if isinstance(index, torch.Tensor): index = index.item() batch_data = dict() From 70a96948f2df769057d01d06492d83d0b618b662 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 13:57:06 -0700 Subject: [PATCH 134/752] WIP --- examples/LLM_plus_GNN.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 2a493049564b..74434cbcf8c6 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -426,8 +426,6 @@ def main(since): val_loss = val_loss / len(val_loader) print(f"Epoch: {epoch + 1}|{num_epochs}: Val Loss: {val_loss}") - print(f'Epoch {epoch + 1} Val Loss {val_loss}') - torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From c6cd039a1c0e38124ec359c73ab3fd3f20a3cdba Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 14:31:09 -0700 Subject: [PATCH 135/752] WIP --- examples/LLM_plus_GNN.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 74434cbcf8c6..45dd0fba9177 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -73,9 +73,10 @@ def compute_accuracy(eval_output): all_recall.append(recall) all_f1.append(f1) - except: # noqa + except Exception as e: # noqa print(f'Label: {label}') print(f'Pred: {pred}') + print("Exception:" e) print('------------------') hit = sum(all_hit) / len(all_hit) precision = sum(all_precision) / len(all_precision) @@ -412,9 +413,8 @@ def main(since): if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] - - print(f"Epoch: {epoch + 1}|{num_epochs}, \ - Train Loss (Epoch Mean): {epoch_loss / len(train_loader)}") + train_loss = epoch_loss / len(train_loader) + print(f"Epoch: {epoch + 1}|{num_epochs}, Train Loss (Epoch Mean): {train_loss}") val_loss = 0. eval_output = [] @@ -424,7 +424,7 @@ def main(since): loss = model(batch) val_loss += loss.item() val_loss = val_loss / len(val_loader) - print(f"Epoch: {epoch + 1}|{num_epochs}: Val Loss: {val_loss}") + print(f"Epoch: {epoch + 1}|{num_epochs}, Val Loss: {val_loss}") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From bd025932f782c1453391f543704df4c4986b1565 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 14:32:03 -0700 Subject: [PATCH 136/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 45dd0fba9177..4f13e59a67d0 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -76,7 +76,7 @@ def compute_accuracy(eval_output): except Exception as e: # noqa print(f'Label: {label}') print(f'Pred: {pred}') - print("Exception:" e) + print("Exception:", e) print('------------------') hit = sum(all_hit) / len(all_hit) precision = sum(all_precision) / len(all_precision) From 4c0a269b034f681cfed77526fe289ccc5686720e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 14:32:21 -0700 Subject: [PATCH 137/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 4f13e59a67d0..23738b3d4dbf 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -73,10 +73,10 @@ def compute_accuracy(eval_output): all_recall.append(recall) all_f1.append(f1) - except Exception as e: # noqa + except Exception as e: print(f'Label: {label}') print(f'Pred: {pred}') - print("Exception:", e) + print(f'Exception: {e}') print('------------------') hit = sum(all_hit) / len(all_hit) precision = sum(all_precision) / len(all_precision) From 5a0bc4560772c05072f3de7368f031d237e0c8b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 21:33:15 +0000 Subject: [PATCH 138/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 23738b3d4dbf..3f6ac373f49e 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -414,7 +414,9 @@ def main(since): if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] train_loss = epoch_loss / len(train_loader) - print(f"Epoch: {epoch + 1}|{num_epochs}, Train Loss (Epoch Mean): {train_loss}") + print( + f"Epoch: {epoch + 1}|{num_epochs}, Train Loss (Epoch Mean): {train_loss}" + ) val_loss = 0. eval_output = [] From bdd4dbe27e059574021306a3a3bdbbd18c9761ac Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 14:46:07 -0700 Subject: [PATCH 139/752] WIP --- examples/LLM_plus_GNN.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 23738b3d4dbf..c68d5d9f0fca 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -396,7 +396,8 @@ def main(since): prep_time = round(time.time() - since, 2) print("Total Prep Time (prep_time) =", prep_time) print("Training beginning...") - loader = tqdm(train_loader, desc="Epoch " + str(epoch + 1)) + epoch_str = f"Epoch: {epoch + 1}|{num_epochs}" + loader = tqdm(train_loader, desc=epoch_str) for step, batch in enumerate(loader): optimizer.zero_grad() loss = model(batch) @@ -414,7 +415,7 @@ def main(since): if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] train_loss = epoch_loss / len(train_loader) - print(f"Epoch: {epoch + 1}|{num_epochs}, Train Loss (Epoch Mean): {train_loss}") + print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") val_loss = 0. eval_output = [] @@ -424,7 +425,7 @@ def main(since): loss = model(batch) val_loss += loss.item() val_loss = val_loss / len(val_loader) - print(f"Epoch: {epoch + 1}|{num_epochs}, Val Loss: {val_loss}") + print(epoch_str + f", Val Loss: {val_loss}") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From 9981874d05f9630cc5035315cd750b5a011454be Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:03:27 -0700 Subject: [PATCH 140/752] WIP --- examples/LLM_plus_GNN.py | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index c68d5d9f0fca..6c19b2b3da23 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -1,11 +1,12 @@ -'''This example implements G-retriever using PyG. +"""This example implements G-retriever using PyG. Original Paper: https://arxiv.org/abs/2402.07630 “G-Retriever significantly reduces hallucinations by 54% compared to the [LLM] baseline“. requirements on top of basic PyG: pip install peft datasets transformers pcst_fast sentencepiece tqdm pandas -''' +""" + import contextlib import gc import math @@ -92,7 +93,7 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): - def __init__(self, graph_type, path, init_prompt): + def __init__(self, llm_model_path: str): super().__init__() self.max_txt_len = 512 self.max_new_tokens = 32 @@ -127,14 +128,14 @@ def __init__(self, graph_type, path, init_prompt): kwargs["max_memory"] = max_mem_dict kwargs["device_map"] = "auto" print("Setting up LLAMA w/ kwargs =", kwargs) - llm_model_path = path + llm_model_path = llm_model_path self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, use_fast=False) self.tokenizer.pad_token_id = 0 self.tokenizer.padding_side = 'left' model = AutoModelForCausalLM.from_pretrained(llm_model_path, - torch_dtype=torch.float16, + torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, **kwargs) @@ -173,17 +174,7 @@ def __init__(self, graph_type, path, init_prompt): self.word_embedding = self.model.model.get_input_embeddings() - def maybe_autocast(self, dtype=torch.bfloat16): - # if on cpu, don't use autocast - # if on gpu, use autocast with dtype if provided, - # otherwise use torch.float16 - enable_autocast = self.model.device != torch.device("cpu") - if enable_autocast: - return torch.cuda.amp.autocast(dtype=dtype) - else: - return contextlib.nullcontext() - - def encode_graphs(self, samples): + def encode_graphs(self, samples: Batch): x = samples.x.to(self.model.device) edge_index = samples.edge_index.long().to(self.model.device) edge_attr = samples.edge_attr.to(self.model.device) @@ -193,7 +184,7 @@ def encode_graphs(self, samples): g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds - def forward(self, samples): + def forward(self, samples: Batch): # encode description, questions and labels batch_size = len(samples.question) questions = self.tokenizer(samples.question, add_special_tokens=False) @@ -257,7 +248,7 @@ def forward(self, samples): label_input_ids = torch.tensor(batch_label_input_ids).to( self.model.device) - with self.maybe_autocast(): + with torch.cuda.amp.autocast(dtype=torch.bfloat16): outputs = self.model( inputs_embeds=inputs_embeds, attention_mask=attention_mask, @@ -267,7 +258,7 @@ def forward(self, samples): return outputs.loss - def inference(self, samples): + def inference(self, samples: Batch): # encode description and questions batch_size = len(samples['question']) questions = self.tokenizer(samples["question"], @@ -318,7 +309,7 @@ def inference(self, samples): attention_mask = torch.tensor(batch_attention_mask).to( self.model.device) - with self.maybe_autocast(): + with torch.cuda.amp.autocast(dtype=torch.bfloat16): outputs = self.model.generate( inputs_embeds=inputs_embeds, max_new_tokens=self.max_new_tokens, @@ -348,7 +339,7 @@ def print_trainable_params(self): return trainable_params, all_param -def main(since): +def main(since: float): seed_everything(42) dataset = WebQSPDataset() @@ -356,8 +347,7 @@ def main(since): # Step 1: Build Node Classification Dataset train_dataset = [dataset[i] for i in idx_split['train']] - val_dataset = [dataset[i] - for i in idx_split['val']] # if dataset[i].x.numel() > 0 + val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, @@ -369,8 +359,7 @@ def main(since): # Step 2: Build Model llm_model_path = "meta-llama/Llama-2-7b-chat-hf" - model = GAT_LLAMA(graph_type=dataset.graph_type, path=llm_model_path, - init_prompt=dataset.prompt) + model = GAT_LLAMA(path=llm_model_path) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] From 594d71171022c21172f23ebaf7917465c4aee5b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:04:59 +0000 Subject: [PATCH 141/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 6c19b2b3da23..e883aff30667 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -134,10 +134,9 @@ def __init__(self, llm_model_path: str): self.tokenizer.pad_token_id = 0 self.tokenizer.padding_side = 'left' - model = AutoModelForCausalLM.from_pretrained(llm_model_path, - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - **kwargs) + model = AutoModelForCausalLM.from_pretrained( + llm_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, + **kwargs) print("Training LLAMA with LORA!") self.model = prepare_model_for_int8_training(model) @@ -404,7 +403,7 @@ def main(since: float): if (step + 1) % grad_steps == 0: lr = optimizer.param_groups[0]["lr"] train_loss = epoch_loss / len(train_loader) - print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") + print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") val_loss = 0. eval_output = [] From fae9a74f1477c1d1ba8a5640ffacf84e10a7e555 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:20:39 -0700 Subject: [PATCH 142/752] WIP --- examples/LLM_plus_GNN.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index e883aff30667..0d88eeb4bc1f 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -357,8 +357,7 @@ def main(since: float): pin_memory=True, shuffle=False) # Step 2: Build Model - llm_model_path = "meta-llama/Llama-2-7b-chat-hf" - model = GAT_LLAMA(path=llm_model_path) + model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf") # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] From 6bbdac64d1fe479c74e72f2a62a3ed2336c04c2d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:28:15 -0700 Subject: [PATCH 143/752] WIP --- examples/LLM_plus_GNN.py | 1 - torch_geometric/datasets/web_qsp_dataset.py | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 0d88eeb4bc1f..75c6819ad20e 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -7,7 +7,6 @@ pip install peft datasets transformers pcst_fast sentencepiece tqdm pandas """ -import contextlib import gc import math import re diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 3232fb4dbe31..f4a827ad4d5f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -5,7 +5,11 @@ import pandas as pd import torch import torch.nn.functional as F -from pcst_fast import pcst_fast +try: + from pcst_fast import pcst_fast + WITH_PCST = True +except ImportError as e: + WITH_PCST = False from torch.utils.data import DataLoader from tqdm import tqdm from transformers import AutoModel, AutoTokenizer @@ -226,7 +230,7 @@ def __init__( root: str = "", force_reload: bool = False, ) -> None: - + assert WITH_PCST, "Please `pip install pcst_fast` to use this dataset." self.prompt = "Please answer the given question." self.graph = None self.graph_type = "Knowledge Graph" From a738c85e393c19df67f33ba9b140ca2be2ac5c11 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:29:20 +0000 Subject: [PATCH 144/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f4a827ad4d5f..63e879d28808 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -5,6 +5,7 @@ import pandas as pd import torch import torch.nn.functional as F + try: from pcst_fast import pcst_fast WITH_PCST = True From 40b21d13f7efc455fd97cc606e017ca332e8f322 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:34:51 -0700 Subject: [PATCH 145/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f4a827ad4d5f..7978c366ac49 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -8,7 +8,7 @@ try: from pcst_fast import pcst_fast WITH_PCST = True -except ImportError as e: +except ImportError as e: # noqa WITH_PCST = False from torch.utils.data import DataLoader from tqdm import tqdm From c370dab0c4db9f7ad87f644cc649b1436bf1d568 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:36:05 +0000 Subject: [PATCH 146/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d46440dfa82d..17ca358caeee 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -9,7 +9,7 @@ try: from pcst_fast import pcst_fast WITH_PCST = True -except ImportError as e: # noqa +except ImportError as e: # noqa WITH_PCST = False from torch.utils.data import DataLoader from tqdm import tqdm From 533d473c903b19f09227005059cff20e63bc1750 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:47:17 -0700 Subject: [PATCH 147/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 28 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d46440dfa82d..4b0751814008 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,6 +1,5 @@ from typing import List, Tuple -import datasets import numpy as np import pandas as pd import torch @@ -13,7 +12,16 @@ WITH_PCST = False from torch.utils.data import DataLoader from tqdm import tqdm -from transformers import AutoModel, AutoTokenizer +try: + from transformers import AutoModel, AutoTokenizer + WITH_TRANSFORMERS = True +except: + WITH_TRANSFORMERS = False +try: + import datasets + WITH_DATASETS = True +except: + WITH_DATASETS = False from torch_geometric.data import Data, InMemoryDataset @@ -231,7 +239,21 @@ def __init__( root: str = "", force_reload: bool = False, ) -> None: - assert WITH_PCST, "Please `pip install pcst_fast` to use this dataset." + missing_imports = False + missing_str = [] + if not WITH_PCST: + missing_str.append('pcst_fast') + missing_imports = True + if not WITH_TRANSFORMERS: + missing_str.append('transformers') + missing_imports = True + if not WITH_DATASETS: + missing_str.append('datasets') + missing_imports = True + if missing_imports: + missing_str = missing_str.join(' ') + error_out = f"`pip install {missing_str}` to use this dataset." + raise ImportError(error_out) self.prompt = "Please answer the given question." self.graph = None self.graph_type = "Knowledge Graph" From 88471777e05dc988a0dbff7a0bf3b9a97b0d1d0a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:50:39 +0000 Subject: [PATCH 148/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0534e7be8639..6b3574fe900f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -12,6 +12,7 @@ WITH_PCST = False from torch.utils.data import DataLoader from tqdm import tqdm + try: from transformers import AutoModel, AutoTokenizer WITH_TRANSFORMERS = True From 9d75682ba1fdbe0c96d87b7a9aac9cf5267518ff Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 15:57:26 -0700 Subject: [PATCH 149/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0534e7be8639..163616f15f3d 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,7 +1,11 @@ from typing import List, Tuple import numpy as np -import pandas as pd +try: + import pandas as pd + WITH_PANDAS = True +except: + WITH_PANDAS = False import torch import torch.nn.functional as F @@ -250,6 +254,9 @@ def __init__( if not WITH_DATASETS: missing_str.append('datasets') missing_imports = True + if not WITH_PANDAS: + missing_str.append('pandas') + missing_imports = True if missing_imports: missing_str = missing_str.join(' ') error_out = f"`pip install {missing_str}` to use this dataset." From 1445052c126d5a355e51512c8b96d23ced93c2f9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:58:36 +0000 Subject: [PATCH 150/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 258716a8edc9..7ba5cbe3de12 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,6 +1,7 @@ from typing import List, Tuple import numpy as np + try: import pandas as pd WITH_PANDAS = True From ace869aeb1e52c7a11f58ca2030e044e1c5a900d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 16:03:28 -0700 Subject: [PATCH 151/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 217 ++++++++++---------- 1 file changed, 108 insertions(+), 109 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 258716a8edc9..4aa9ee377f8f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -4,7 +4,7 @@ try: import pandas as pd WITH_PANDAS = True -except: +except ImportError as e: # noqa WITH_PANDAS = False import torch import torch.nn.functional as F @@ -20,123 +20,17 @@ try: from transformers import AutoModel, AutoTokenizer WITH_TRANSFORMERS = True -except: +except ImportError as e: # noqa WITH_TRANSFORMERS = False try: import datasets WITH_DATASETS = True -except: +except ImportError as e: # noqa WITH_DATASETS = False from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, - textual_nodes: pd.DataFrame, - textual_edges: pd.DataFrame, topk: int = 3, - topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: - # from G-Retriever repo - c = 0.01 - if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - graph = Data(x=graph.x, edge_index=graph.edge_index, - edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) - return graph, desc - - root = -1 # unrooted - num_clusters = 1 - pruning = "gw" - verbosity_level = 0 - if topk > 0: - n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) - topk = min(topk, graph.num_nodes) - _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) - - n_prizes = torch.zeros_like(n_prizes) - n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() - else: - n_prizes = torch.zeros(graph.num_nodes) - - if topk_e > 0: - e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) - topk_e = min(topk_e, e_prizes.unique().size(0)) - - topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) - e_prizes[e_prizes < topk_e_values[-1]] = 0.0 - last_topk_e_value = topk_e - for k in range(topk_e): - indices = e_prizes == topk_e_values[k] - value = min((topk_e - k) / sum(indices), last_topk_e_value - c) - e_prizes[indices] = value - last_topk_e_value = value - # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) - else: - e_prizes = torch.zeros(graph.num_edges) - - costs = [] - edges = [] - vritual_n_prizes = [] - virtual_edges = [] - virtual_costs = [] - mapping_n = {} - mapping_e = {} - for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): - prize_e = e_prizes[i] - if prize_e <= cost_e: - mapping_e[len(edges)] = i - edges.append((src, dst)) - costs.append(cost_e - prize_e) - else: - virtual_node_id = graph.num_nodes + len(vritual_n_prizes) - mapping_n[virtual_node_id] = i - virtual_edges.append((src, virtual_node_id)) - virtual_edges.append((virtual_node_id, dst)) - virtual_costs.append(0) - virtual_costs.append(0) - vritual_n_prizes.append(prize_e - cost_e) - - prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) - num_edges = len(edges) - if len(virtual_costs) > 0: - costs = np.array(costs + virtual_costs) - edges = np.array(edges + virtual_edges) - - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, - pruning, verbosity_level) - - selected_nodes = vertices[vertices < graph.num_nodes] - selected_edges = [mapping_e[e] for e in edges if e < num_edges] - virtual_vertices = vertices[vertices >= graph.num_nodes] - if len(virtual_vertices) > 0: - virtual_vertices = vertices[vertices >= graph.num_nodes] - virtual_edges = [mapping_n[i] for i in virtual_vertices] - selected_edges = np.array(selected_edges + virtual_edges) - - edge_index = graph.edge_index[:, selected_edges] - selected_nodes = np.unique( - np.concatenate( - [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) - - n = textual_nodes.iloc[selected_nodes] - e = textual_edges.iloc[selected_edges] - desc = n.to_csv(index=False) + "\n" + e.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - - mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} - - x = graph.x[selected_nodes] - edge_attr = graph.edge_attr[selected_edges] - src = [mapping[i] for i in edge_index[0].tolist()] - dst = [mapping[i] for i in edge_index[1].tolist()] - edge_index = torch.LongTensor([src, dst]) - data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(selected_nodes)) - - return data, desc - - class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids: torch.Tensor = None, attention_mask: torch.Tensor = None): @@ -271,6 +165,111 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) + def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, + textual_nodes: pd.DataFrame, + textual_edges: pd.DataFrame, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: + # from G-Retriever repo + c = 0.01 + if len(textual_nodes) == 0 or len(textual_edges) == 0: + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + graph = Data(x=graph.x, edge_index=graph.edge_index, + edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + return graph, desc + + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 + if topk > 0: + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) + topk = min(topk, graph.num_nodes) + _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) + + n_prizes = torch.zeros_like(n_prizes) + n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() + else: + n_prizes = torch.zeros(graph.num_nodes) + + if topk_e > 0: + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + topk_e = min(topk_e, e_prizes.unique().size(0)) + + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + e_prizes[e_prizes < topk_e_values[-1]] = 0.0 + last_topk_e_value = topk_e + for k in range(topk_e): + indices = e_prizes == topk_e_values[k] + value = min((topk_e - k) / sum(indices), last_topk_e_value - c) + e_prizes[indices] = value + last_topk_e_value = value + # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) + else: + e_prizes = torch.zeros(graph.num_edges) + + costs = [] + edges = [] + vritual_n_prizes = [] + virtual_edges = [] + virtual_costs = [] + mapping_n = {} + mapping_e = {} + for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + prize_e = e_prizes[i] + if prize_e <= cost_e: + mapping_e[len(edges)] = i + edges.append((src, dst)) + costs.append(cost_e - prize_e) + else: + virtual_node_id = graph.num_nodes + len(vritual_n_prizes) + mapping_n[virtual_node_id] = i + virtual_edges.append((src, virtual_node_id)) + virtual_edges.append((virtual_node_id, dst)) + virtual_costs.append(0) + virtual_costs.append(0) + vritual_n_prizes.append(prize_e - cost_e) + + prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) + num_edges = len(edges) + if len(virtual_costs) > 0: + costs = np.array(costs + virtual_costs) + edges = np.array(edges + virtual_edges) + + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) + + selected_nodes = vertices[vertices < graph.num_nodes] + selected_edges = [mapping_e[e] for e in edges if e < num_edges] + virtual_vertices = vertices[vertices >= graph.num_nodes] + if len(virtual_vertices) > 0: + virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges + virtual_edges) + + edge_index = graph.edge_index[:, selected_edges] + selected_nodes = np.unique( + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + + n = textual_nodes.iloc[selected_nodes] + e = textual_edges.iloc[selected_edges] + desc = n.to_csv(index=False) + "\n" + e.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + + mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + + x = graph.x[selected_nodes] + edge_attr = graph.edge_attr[selected_edges] + src = [mapping[i] for i in edge_index[0].tolist()] + dst = [mapping[i] for i in edge_index[1].tolist()] + edge_index = torch.LongTensor([src, dst]) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) + + return data, desc + @property def raw_file_names(self) -> List[str]: return [] From 47a3b70058057ea78bfa9dc66309142c671e0566 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:04:37 +0000 Subject: [PATCH 152/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a34f52e83534..670714a91477 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -174,8 +174,9 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, # from G-Retriever repo c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) + desc = textual_nodes.to_csv( + index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) graph = Data(x=graph.x, edge_index=graph.edge_index, edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc @@ -195,10 +196,12 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, n_prizes = torch.zeros(graph.num_nodes) if topk_e > 0: - e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, + graph.edge_attr) topk_e = min(topk_e, e_prizes.unique().size(0)) - topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, + largest=True) e_prizes[e_prizes < topk_e_values[-1]] = 0.0 last_topk_e_value = topk_e for k in range(topk_e): @@ -252,7 +255,8 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, edge_index = graph.edge_index[:, selected_edges] selected_nodes = np.unique( np.concatenate( - [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + [selected_nodes, edge_index[0].numpy(), + edge_index[1].numpy()])) n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] @@ -270,7 +274,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, num_nodes=len(selected_nodes)) return data, desc - + @property def raw_file_names(self) -> List[str]: return [] From b29523d975dd5368f5896a8a295a64fb6818e95f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 16:07:32 -0700 Subject: [PATCH 153/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a34f52e83534..0877e70c545c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -171,7 +171,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_edges: pd.DataFrame, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: - # from G-Retriever repo + # from original G-Retriever work c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( @@ -346,7 +346,7 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + psct_subgraph, desc = WebQSPDataset.retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question From 638c7a4846532667ebac6d7fe8641fe38b96ff7a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:08:41 +0000 Subject: [PATCH 154/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a211fd6ddf3a..5a61b6940092 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -350,9 +350,9 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = WebQSPDataset.retrieval_via_pcst(raw_graph, q_embs[index], - nodes, edges, topk=3, - topk_e=5, cost_e=0.5) + psct_subgraph, desc = WebQSPDataset.retrieval_via_pcst( + raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, + cost_e=0.5) psct_subgraph["question"] = question psct_subgraph["label"] = label psct_subgraph["desc"] = desc From 19fecfc11dba86a5bcb0cd57a9f912486a0a8899 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 11 Mar 2024 16:16:01 -0700 Subject: [PATCH 155/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a211fd6ddf3a..3c662a6256d8 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -167,8 +167,8 @@ def __init__( self.load(self.processed_paths[0]) def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, - textual_nodes: pd.DataFrame, - textual_edges: pd.DataFrame, topk: int = 3, + textual_nodes, + textual_edges, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work From 085a738f29df7a515b6b330bfa0d00cc1683ec03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:17:35 +0000 Subject: [PATCH 156/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a4ba83f31574..37811754538f 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -166,10 +166,8 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, - textual_nodes, - textual_edges, topk: int = 3, - topk_e: int = 3, + def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes, + textual_edges, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work c = 0.01 From 237d9b6929629d7ae55590a6405559c96b334f04 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 10:56:59 -0700 Subject: [PATCH 157/752] WIP --- torch_geometric/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index d5ffe5bf8bdc..802bfe8c6019 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -108,7 +108,7 @@ import torch_geometric.datasets.utils homo_datasets = [ - 'WebQSPDataset' + 'WebQSPDataset', 'KarateClub', 'TUDataset', 'GNNBenchmarkDataset', From 30ec99fccc6926d6a09ced4ec83b9524e55e2af3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 11:10:41 -0700 Subject: [PATCH 158/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 37811754538f..ef0faa5dfa25 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import List, Tuple +from typing import List, Tuple, Optional import numpy as np @@ -33,8 +33,8 @@ class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids: torch.Tensor = None, - attention_mask: torch.Tensor = None): + def __init__(self, input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None) -> None: super().__init__() self.data = { "input_ids": input_ids, @@ -81,7 +81,7 @@ def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor): def sbert_text2embedding(model: Sentence_Transformer, tokenizer: torch.nn.Module, device: torch.device, - text: List[str]) -> List[torch.Tensor]: + text: List[str]) -> torch.Tensor: try: encoding = tokenizer(text, padding=True, truncation=True, return_tensors="pt") @@ -92,7 +92,7 @@ def sbert_text2embedding(model: Sentence_Transformer, dataloader = DataLoader(dataset, batch_size=256, shuffle=False) # Placeholder for storing the embeddings - all_embeddings = [] + all_embeddings_list = [] # Iterate through batches with torch.no_grad(): @@ -106,10 +106,10 @@ def sbert_text2embedding(model: Sentence_Transformer, att_mask=batch["att_mask"]) # Append the embeddings to the list - all_embeddings.append(embeddings) + all_embeddings_list.append(embeddings) # Concatenate the embeddings from all batches - all_embeddings = torch.cat(all_embeddings, dim=0).cpu() + all_embeddings = torch.cat(all_embeddings_list, dim=0).cpu() except: # noqa print( "SBERT text embedding failed, returning torch.zeros((0, 1024))...") @@ -166,10 +166,11 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes, - textual_edges, topk: int = 3, topk_e: int = 3, + def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, + textual_edges: pd.DataFrame, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work + # https://arxiv.org/abs/2402.07630 c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( @@ -348,7 +349,7 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = WebQSPDataset.retrieval_via_pcst( + psct_subgraph, desc = self.retrieval_via_pcst( raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question From 9ddc8d301e085a675eeab1fba6165744fa230d4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:11:45 +0000 Subject: [PATCH 159/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ef0faa5dfa25..071a920936d6 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple import numpy as np @@ -166,8 +166,10 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, - textual_edges: pd.DataFrame, topk: int = 3, topk_e: int = 3, + def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, + textual_nodes: pd.DataFrame, + textual_edges: pd.DataFrame, topk: int = 3, + topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work # https://arxiv.org/abs/2402.07630 From 674b458d13f88f0cd4fff0df41eebfd4634b6f25 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:04:24 -0700 Subject: [PATCH 160/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ef0faa5dfa25..d5af3d31668b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -4,8 +4,10 @@ try: import pandas as pd + from pandas import DataFrame as df WITH_PANDAS = True except ImportError as e: # noqa + df = None WITH_PANDAS = False import torch import torch.nn.functional as F @@ -166,8 +168,8 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, textual_nodes: pd.DataFrame, - textual_edges: pd.DataFrame, topk: int = 3, topk_e: int = 3, + def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, textual_nodes: df, + textual_edges: df, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work # https://arxiv.org/abs/2402.07630 From 07882b4adfc4c78735e248e47dc350d3762f118a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 20:06:04 +0000 Subject: [PATCH 161/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e2c32715b40e..ed92e0ddede7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -168,8 +168,9 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, textual_nodes: df, - textual_edges: df, topk: int = 3, topk_e: int = 3, + def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, + textual_nodes: df, textual_edges: df, topk: int = 3, + topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work # https://arxiv.org/abs/2402.07630 From 5126a9f61dc4305e782a11503222d9984197de17 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:08:49 -0700 Subject: [PATCH 162/752] WIP --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 75c6819ad20e..61564218f26b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -114,10 +114,10 @@ def __init__(self, llm_model_path: str): gpus_2_use_4_llm += 1 # We want to use the minimum number of GPUs that LLM can fit on # this is to minimize the need for interGPU communications - if mem_total >= 80: + if mem_total >= 75: break - assert mem_total >= 80, \ + assert mem_total >= 75, \ "Need ~80GB of GPU RAM across all GPUs on device, only " \ + str(mem_total) + "GB available across " + str(avail_gpus) \ + " GPUs" From f1b5a7fd1633c2f6193ea2372e3f6f659c4f8d7c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:13:02 -0700 Subject: [PATCH 163/752] WIP --- examples/LLM_plus_GNN.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index 61564218f26b..dd1dda083437 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -117,10 +117,11 @@ def __init__(self, llm_model_path: str): if mem_total >= 75: break - assert mem_total >= 75, \ - "Need ~80GB of GPU RAM across all GPUs on device, only " \ - + str(mem_total) + "GB available across " + str(avail_gpus) \ - + " GPUs" + if mem_total < 75: + print("~75GB of GPU RAM recommended across all GPUs on device, \ + only " + str(mem_total) + "GB available across " + str(avail_gpus) \ + + " GPUs") + for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" From c5de2c06c36bc8658ca68fa392a7a3bdd9bcd434 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 20:14:05 +0000 Subject: [PATCH 164/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/LLM_plus_GNN.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index dd1dda083437..6f1e7ce9985b 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -119,9 +119,8 @@ def __init__(self, llm_model_path: str): if mem_total < 75: print("~75GB of GPU RAM recommended across all GPUs on device, \ - only " + str(mem_total) + "GB available across " + str(avail_gpus) \ + only " + str(mem_total) + "GB available across " + str(avail_gpus) \ + " GPUs") - for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" From 0c623846461b5bb7fbc3fd88da1c2af7122a6105 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:19:51 -0700 Subject: [PATCH 165/752] WIP --- examples/LLM_plus_GNN.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index dd1dda083437..fed8bfcd9b86 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -114,13 +114,9 @@ def __init__(self, llm_model_path: str): gpus_2_use_4_llm += 1 # We want to use the minimum number of GPUs that LLM can fit on # this is to minimize the need for interGPU communications + # 75 GB VRAM in total is recommended if mem_total >= 75: break - - if mem_total < 75: - print("~75GB of GPU RAM recommended across all GPUs on device, \ - only " + str(mem_total) + "GB available across " + str(avail_gpus) \ - + " GPUs") for i in range(gpus_2_use_4_llm): From d48005040133004603331499e7a08a4ef75880c9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:21:31 -0700 Subject: [PATCH 166/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ed92e0ddede7..9eb4f262e3c1 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -211,7 +211,6 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, value = min((topk_e - k) / sum(indices), last_topk_e_value - c) e_prizes[indices] = value last_topk_e_value = value - # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) else: e_prizes = torch.zeros(graph.num_edges) From 3908055be392dd03bdb88aa544c35a6cadc76806 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 13:30:59 -0700 Subject: [PATCH 167/752] WIP --- examples/LLM_plus_GNN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index acf471f21124..a2a62fd7b77e 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -114,7 +114,7 @@ def __init__(self, llm_model_path: str): gpus_2_use_4_llm += 1 # We want to use the minimum number of GPUs that LLM can fit on # this is to minimize the need for interGPU communications - # 75 GB VRAM in total is recommended + # >= 75 GB VRAM in total is recommended if mem_total >= 75: break From b2b8710d2532bfec8481e002c001d49ba8e0b4a1 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 12 Mar 2024 17:18:09 -0700 Subject: [PATCH 168/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 9eb4f262e3c1..bfc0bafcc5ec 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Dict import numpy as np @@ -46,7 +46,7 @@ def __init__(self, input_ids: Optional[torch.Tensor] = None, def __len__(self): return self.data["input_ids"].size(0) - def __getitem__(self, index: int): + def __getitem__(self, index: int) -> Dict[str, torch.Tensor]: if isinstance(index, torch.Tensor): index = index.item() batch_data = dict() @@ -63,14 +63,14 @@ def __init__(self, pretrained_repo: str) -> None: self.bert_model = AutoModel.from_pretrained(pretrained_repo) def mean_pooling(self, token_embeddings: torch.Tensor, - attention_mask: torch.Tensor): + attention_mask: torch.Tensor) -> torch.Tensor: data_type = token_embeddings.dtype input_mask_expanded = attention_mask.unsqueeze(-1).expand( token_embeddings.size()).to(data_type) return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) - def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor): + def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor) -> torch.Tensor: bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) From 6e093ad2dcfb22c7c4ad190050b397ef5e0fc23a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:19:13 +0000 Subject: [PATCH 169/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index bfc0bafcc5ec..0f71a963a176 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple, Dict +from typing import Dict, List, Optional, Tuple import numpy as np @@ -70,7 +70,8 @@ def mean_pooling(self, token_embeddings: torch.Tensor, return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) - def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor) -> torch.Tensor: + def forward(self, input_ids: torch.Tensor, + att_mask: torch.Tensor) -> torch.Tensor: bert_out = self.bert_model(input_ids=input_ids, attention_mask=att_mask) From becdae40970e122b644146f7f7b3246b0f19b5bf Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 13:34:45 -0700 Subject: [PATCH 170/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index bfc0bafcc5ec..b34ee243a2c4 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -156,7 +156,7 @@ def __init__( missing_str.append('pandas') missing_imports = True if missing_imports: - missing_str = missing_str.join(' ') + missing_str = ' '.join(missing_str) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) self.prompt = "Please answer the given question." From bec2aa81580c7f97168c53ef32bfa7d4d7bca45d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 13:46:49 -0700 Subject: [PATCH 171/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 6eab2716482f..7393e470aa4d 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -176,6 +176,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 + num_nodes = graph.num_nodes if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( index=False) + "\n" + textual_edges.to_csv( @@ -190,13 +191,13 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) - topk = min(topk, graph.num_nodes) + topk = min(topk, num_nodes) # noqa _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) n_prizes = torch.zeros_like(n_prizes) n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() else: - n_prizes = torch.zeros(graph.num_nodes) + n_prizes = torch.zeros(num_nodes) if topk_e > 0: e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, @@ -229,7 +230,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, edges.append((src, dst)) costs.append(cost_e - prize_e) else: - virtual_node_id = graph.num_nodes + len(vritual_n_prizes) + virtual_node_id = num_nodes + len(vritual_n_prizes) mapping_n[virtual_node_id] = i virtual_edges.append((src, virtual_node_id)) virtual_edges.append((virtual_node_id, dst)) @@ -246,11 +247,11 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) - selected_nodes = vertices[vertices < graph.num_nodes] + selected_nodes = vertices[vertices < num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] - virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_vertices = vertices[vertices >= num_nodes] if len(virtual_vertices) > 0: - virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_vertices = vertices[vertices >= num_nodes] virtual_edges = [mapping_n[i] for i in virtual_vertices] selected_edges = np.array(selected_edges + virtual_edges) From 9f18cfba64d8bde3667296b589265ce69472f6dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 20:48:02 +0000 Subject: [PATCH 172/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 7393e470aa4d..deb8e72bf2f5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -191,7 +191,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) - topk = min(topk, num_nodes) # noqa + topk = min(topk, num_nodes) # noqa _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) n_prizes = torch.zeros_like(n_prizes) From 9a4ebb11993fccc2fc00b6f474b88f6189dd59a8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 14:01:27 -0700 Subject: [PATCH 173/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 7393e470aa4d..ec5d75f85f35 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -143,21 +143,21 @@ def __init__( force_reload: bool = False, ) -> None: missing_imports = False - missing_str = [] + missing_str_list = [] if not WITH_PCST: - missing_str.append('pcst_fast') + missing_str_list.append('pcst_fast') missing_imports = True if not WITH_TRANSFORMERS: - missing_str.append('transformers') + missing_str_list.append('transformers') missing_imports = True if not WITH_DATASETS: - missing_str.append('datasets') + missing_str_list.append('datasets') missing_imports = True if not WITH_PANDAS: - missing_str.append('pandas') + missing_str_list.append('pandas') missing_imports = True if missing_imports: - missing_str = ' '.join(missing_str) + missing_str = ' '.join(missing_str_list) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) self.prompt = "Please answer the given question." @@ -176,22 +176,26 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 - num_nodes = graph.num_nodes + num_nodes = (int) graph.num_nodes + num_edges = (int) graph.num_edges + e_idx = (torch.Tensor) graph.edge_index + e_attr = (torch.Tensor) graph.edge_attr + node_feat = (torch.Tensor) graph.x if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( index=False) + "\n" + textual_edges.to_csv( index=False, columns=["src", "edge_attr", "dst"]) - graph = Data(x=graph.x, edge_index=graph.edge_index, - edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) - return graph, desc + new_graph = Data(x=node_feat, edge_index=e_idx, + edge_attr=e_attr, num_nodes=num_nodes) + return new_graph, desc root = -1 # unrooted num_clusters = 1 pruning = "gw" verbosity_level = 0 if topk > 0: - n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) - topk = min(topk, num_nodes) # noqa + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, node_feat) + topk = min(topk, num_nodes) _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) n_prizes = torch.zeros_like(n_prizes) @@ -201,7 +205,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, if topk_e > 0: e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, - graph.edge_attr) + e_attr) topk_e = min(topk_e, e_prizes.unique().size(0)) topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, @@ -214,7 +218,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, e_prizes[indices] = value last_topk_e_value = value else: - e_prizes = torch.zeros(graph.num_edges) + e_prizes = torch.zeros(num_edges) costs = [] edges = [] @@ -223,7 +227,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_costs = [] mapping_n = {} mapping_e = {} - for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + for i, (src, dst) in enumerate(e_idx.T.numpy()): prize_e = e_prizes[i] if prize_e <= cost_e: mapping_e[len(edges)] = i From 1083bfc24f80e704bba3d56902bcf178baccd3cd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 14:40:27 -0700 Subject: [PATCH 174/752] WIP --- torch_geometric/datasets/web_qsp_dataset.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index ec5d75f85f35..21d9ca46888c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -176,11 +176,12 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 - num_nodes = (int) graph.num_nodes - num_edges = (int) graph.num_edges - e_idx = (torch.Tensor) graph.edge_index - e_attr = (torch.Tensor) graph.edge_attr - node_feat = (torch.Tensor) graph.x + # explicit casting for linting + num_nodes = int(graph.num_nodes) + num_edges = int(graph.num_edges) + e_idx = torch.Tensor(graph.edge_index) + e_attr = torch.Tensor(graph.edge_attr) + node_feat = torch.Tensor(graph.x) if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( index=False) + "\n" + textual_edges.to_csv( From 6cf80564b2b646b8cc9aaee2f2553de2e6bc2bce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 21:41:31 +0000 Subject: [PATCH 175/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 21d9ca46888c..f40654cc994b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -186,8 +186,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, desc = textual_nodes.to_csv( index=False) + "\n" + textual_edges.to_csv( index=False, columns=["src", "edge_attr", "dst"]) - new_graph = Data(x=node_feat, edge_index=e_idx, - edge_attr=e_attr, num_nodes=num_nodes) + new_graph = Data(x=node_feat, edge_index=e_idx, edge_attr=e_attr, + num_nodes=num_nodes) return new_graph, desc root = -1 # unrooted @@ -205,8 +205,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, n_prizes = torch.zeros(num_nodes) if topk_e > 0: - e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, - e_attr) + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, e_attr) topk_e = min(topk_e, e_prizes.unique().size(0)) topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, From 723d6635c1f9360a06b495c7cd18394689ced401 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:03:50 -0700 Subject: [PATCH 176/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 21d9ca46888c..353646fb4de3 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -43,7 +43,7 @@ def __init__(self, input_ids: Optional[torch.Tensor] = None, "att_mask": attention_mask, } - def __len__(self): + def __len__(self) -> int: return self.data["input_ids"].size(0) def __getitem__(self, index: int) -> Dict[str, torch.Tensor]: @@ -221,9 +221,9 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, else: e_prizes = torch.zeros(num_edges) - costs = [] - edges = [] - vritual_n_prizes = [] + cost_list = [] + edge_list = [] + virtual_n_prizes = [] virtual_edges = [] virtual_costs = [] mapping_n = {} @@ -231,11 +231,11 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, for i, (src, dst) in enumerate(e_idx.T.numpy()): prize_e = e_prizes[i] if prize_e <= cost_e: - mapping_e[len(edges)] = i - edges.append((src, dst)) - costs.append(cost_e - prize_e) + mapping_e[len(edge_list)] = i + edge_list.append((src, dst)) + cost_list.append(cost_e - prize_e) else: - virtual_node_id = num_nodes + len(vritual_n_prizes) + virtual_node_id = num_nodes + len(virtual_n_prizes) mapping_n[virtual_node_id] = i virtual_edges.append((src, virtual_node_id)) virtual_edges.append((virtual_node_id, dst)) @@ -243,11 +243,11 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_costs.append(0) vritual_n_prizes.append(prize_e - cost_e) - prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) + prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) num_edges = len(edges) if len(virtual_costs) > 0: - costs = np.array(costs + virtual_costs) - edges = np.array(edges + virtual_edges) + costs = np.array(cost_list + virtual_costs) + edges = np.array(edge_list + virtual_edges) vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) From cc3a9620fda16794909c9f5d7da19b1d71e1d704 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:05:06 -0700 Subject: [PATCH 177/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 6bb1366c384f..e6ff4f2e31b8 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -259,7 +259,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_edges = [mapping_n[i] for i in virtual_vertices] selected_edges = np.array(selected_edges + virtual_edges) - edge_index = graph.edge_index[:, selected_edges] + edge_index = e_idx[:, selected_edges] selected_nodes = np.unique( np.concatenate( [selected_nodes, edge_index[0].numpy(), @@ -272,8 +272,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} - x = graph.x[selected_nodes] - edge_attr = graph.edge_attr[selected_edges] + x = node_feat[selected_nodes] + edge_attr = e_attr[selected_edges] src = [mapping[i] for i in edge_index[0].tolist()] dst = [mapping[i] for i in edge_index[1].tolist()] edge_index = torch.LongTensor([src, dst]) From 699f6a13f0f7ef370d327c6b45595c6231b78493 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:21:45 -0700 Subject: [PATCH 178/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e6ff4f2e31b8..dc1dd167e23b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -240,10 +240,10 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_edges.append((virtual_node_id, dst)) virtual_costs.append(0) virtual_costs.append(0) - vritual_n_prizes.append(prize_e - cost_e) + virtual_n_prizes.append(prize_e - cost_e) prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) - num_edges = len(edges) + num_edges = len(edge_list) if len(virtual_costs) > 0: costs = np.array(cost_list + virtual_costs) edges = np.array(edge_list + virtual_edges) From dcd888f9dd07c7d49ed003d306994b8a74b85930 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:39:04 -0700 Subject: [PATCH 179/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index dc1dd167e23b..5c6bf1712c20 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -35,8 +35,7 @@ class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None) -> None: + def __init__(self, input_ids: torch.Tensor, torch.Tensor) -> None: super().__init__() self.data = { "input_ids": input_ids, @@ -177,10 +176,10 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # https://arxiv.org/abs/2402.07630 c = 0.01 # explicit casting for linting - num_nodes = int(graph.num_nodes) - num_edges = int(graph.num_edges) - e_idx = torch.Tensor(graph.edge_index) - e_attr = torch.Tensor(graph.edge_attr) + num_nodes: int = graph.num_nodes + num_edges: int = graph.num_edges + e_idx: torch.Tensor = graph.edge_index + e_attr: torch.Tensor = graph.edge_attr node_feat = torch.Tensor(graph.x) if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( From ebb5efd598e880d4372624967a463631c109efff Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:43:45 -0700 Subject: [PATCH 180/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5c6bf1712c20..d1e04dc294b7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -35,7 +35,8 @@ class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids: torch.Tensor, torch.Tensor) -> None: + def __init__(self, input_ids: torch.Tensor, + attention_mask: torch.Tensor) -> None: super().__init__() self.data = { "input_ids": input_ids, From d14cf940a1b084ccdb61f8b1f3ce0b1bd36d5150 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 22:44:48 +0000 Subject: [PATCH 181/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d1e04dc294b7..4051a5d70f29 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -36,7 +36,7 @@ class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids: torch.Tensor, - attention_mask: torch.Tensor) -> None: + attention_mask: torch.Tensor) -> None: super().__init__() self.data = { "input_ids": input_ids, From f8a983d9de163241397cd352cfc954d2e975a932 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:47:53 -0700 Subject: [PATCH 182/752] linting --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d1e04dc294b7..b6438c6ecb7b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Tuple import numpy as np From 7dda39b43adb51edfb525b707c22d8f8cdbe1aa8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 15:59:31 -0700 Subject: [PATCH 183/752] linting is super strict... --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d5c72cb1c2ad..d443856e8e68 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -221,7 +221,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, e_prizes = torch.zeros(num_edges) cost_list = [] - edge_list = [] + edge_list: List[np.array] = [] virtual_n_prizes = [] virtual_edges = [] virtual_costs = [] @@ -256,8 +256,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_vertices = vertices[vertices >= num_nodes] if len(virtual_vertices) > 0: virtual_vertices = vertices[vertices >= num_nodes] - virtual_edges = [mapping_n[i] for i in virtual_vertices] - selected_edges = np.array(selected_edges + virtual_edges) + new_virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges + new_virtual_edges) edge_index = e_idx[:, selected_edges] selected_nodes = np.unique( @@ -322,7 +322,7 @@ def process(self) -> None: if index == 1: quit() data_i = self.raw_dataset[index] - raw_nodes = {} + raw_nodes: Dict[str, int] = {} raw_edges = [] for tri in data_i["graph"]: h, r, t = tri From f045626338e9f5791021337185e78d43d152e809 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 16:19:18 -0700 Subject: [PATCH 184/752] linting is super strict... --- torch_geometric/datasets/web_qsp_dataset.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index d443856e8e68..eb81c149e2ea 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -221,7 +221,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, e_prizes = torch.zeros(num_edges) cost_list = [] - edge_list: List[np.array] = [] + edge_list = [] virtual_n_prizes = [] virtual_edges = [] virtual_costs = [] @@ -257,23 +257,23 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, if len(virtual_vertices) > 0: virtual_vertices = vertices[vertices >= num_nodes] new_virtual_edges = [mapping_n[i] for i in virtual_vertices] - selected_edges = np.array(selected_edges + new_virtual_edges) + new_selected_edges = np.array(selected_edges + new_virtual_edges) - edge_index = e_idx[:, selected_edges] - selected_nodes = np.unique( + edge_index = e_idx[:, new_selected_edges] + new_selected_nodes = np.unique( np.concatenate( [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) - n = textual_nodes.iloc[selected_nodes] - e = textual_edges.iloc[selected_edges] + n = textual_nodes.iloc[new_selected_nodes] + e = textual_edges.iloc[new_selected_edges] desc = n.to_csv(index=False) + "\n" + e.to_csv( index=False, columns=["src", "edge_attr", "dst"]) - mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + mapping = {n: i for i, n in enumerate(new_selected_nodes.tolist())} - x = node_feat[selected_nodes] - edge_attr = e_attr[selected_edges] + x = node_feat[new_selected_nodes] + edge_attr = e_attr[new_selected_edges] src = [mapping[i] for i in edge_index[0].tolist()] dst = [mapping[i] for i in edge_index[1].tolist()] edge_index = torch.LongTensor([src, dst]) From e39d89be0f9b58ff3b099dc571d80a5f08db0f67 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 16:33:33 -0700 Subject: [PATCH 185/752] linting is super strict... --- torch_geometric/datasets/web_qsp_dataset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index eb81c149e2ea..8756135a2ae3 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -177,10 +177,10 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # https://arxiv.org/abs/2402.07630 c = 0.01 # explicit casting for linting - num_nodes: int = graph.num_nodes - num_edges: int = graph.num_edges - e_idx: torch.Tensor = graph.edge_index - e_attr: torch.Tensor = graph.edge_attr + num_nodes: int = graph.num_nodes # type: ignore + num_edges: int = graph.num_edges # type: ignore + e_idx: torch.Tensor = graph.edge_index # type: ignore + e_attr: torch.Tensor = graph.edge_attr # type: ignore node_feat = torch.Tensor(graph.x) if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( @@ -221,8 +221,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, e_prizes = torch.zeros(num_edges) cost_list = [] - edge_list = [] - virtual_n_prizes = [] + edge_list = [] # type: ignore + virtual_n_prizes = [] # type: ignore virtual_edges = [] virtual_costs = [] mapping_n = {} From 6e1dc71c6e0c36b49b8437acff960757d6e2367e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 23:37:31 +0000 Subject: [PATCH 186/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 8756135a2ae3..157c41cad962 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -177,10 +177,10 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, # https://arxiv.org/abs/2402.07630 c = 0.01 # explicit casting for linting - num_nodes: int = graph.num_nodes # type: ignore - num_edges: int = graph.num_edges # type: ignore - e_idx: torch.Tensor = graph.edge_index # type: ignore - e_attr: torch.Tensor = graph.edge_attr # type: ignore + num_nodes: int = graph.num_nodes # type: ignore + num_edges: int = graph.num_edges # type: ignore + e_idx: torch.Tensor = graph.edge_index # type: ignore + e_attr: torch.Tensor = graph.edge_attr # type: ignore node_feat = torch.Tensor(graph.x) if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv( @@ -221,8 +221,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, e_prizes = torch.zeros(num_edges) cost_list = [] - edge_list = [] # type: ignore - virtual_n_prizes = [] # type: ignore + edge_list = [] # type: ignore + virtual_n_prizes = [] # type: ignore virtual_edges = [] virtual_costs = [] mapping_n = {} From 80da31670cbdf6cc0d65799d6e55631b101cc1cf Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 16:41:45 -0700 Subject: [PATCH 187/752] linting is super strict... --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 157c41cad962..a1d77fb59248 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -178,7 +178,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, c = 0.01 # explicit casting for linting num_nodes: int = graph.num_nodes # type: ignore - num_edges: int = graph.num_edges # type: ignore + num_edges: int = graph.num_edges e_idx: torch.Tensor = graph.edge_index # type: ignore e_attr: torch.Tensor = graph.edge_attr # type: ignore node_feat = torch.Tensor(graph.x) From 107a806553766d7f1f12a6af1a89ca89b7d5f4a9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 17:16:50 -0700 Subject: [PATCH 188/752] linting is super stricccccbldentkureckegtvjdftneblhtdikldrhg --- torch_geometric/datasets/web_qsp_dataset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a1d77fb59248..c11d555a56ef 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -319,8 +319,6 @@ def process(self) -> None: self.questions) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): - if index == 1: - quit() data_i = self.raw_dataset[index] raw_nodes: Dict[str, int] = {} raw_edges = [] From d98461c99f48e42ea33956403ae0b7104e835afd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 17:20:23 -0700 Subject: [PATCH 189/752] linting is super stricccccbldentkureckegtvjdftneblhtdikldrhg --- torch_geometric/datasets/web_qsp_dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index c11d555a56ef..23002279b51d 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -247,6 +247,9 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, if len(virtual_costs) > 0: costs = np.array(cost_list + virtual_costs) edges = np.array(edge_list + virtual_edges) + else: + costs = cost_list + edges = edge_list vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) From ec99363606fb14fdcc6edf30da569259364aa7a3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 17:24:13 -0700 Subject: [PATCH 190/752] linting is super stricccccbldentkureckegtvjdftneblhtdikldrhg --- torch_geometric/datasets/web_qsp_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 23002279b51d..9ca9022370ea 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -261,6 +261,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, virtual_vertices = vertices[vertices >= num_nodes] new_virtual_edges = [mapping_n[i] for i in virtual_vertices] new_selected_edges = np.array(selected_edges + new_virtual_edges) + else: + new_selected_edges = selected_edges edge_index = e_idx[:, new_selected_edges] new_selected_nodes = np.unique( From 36e14813d6c8483e85ec1e135633c1013029b5cd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 13 Mar 2024 17:43:10 -0700 Subject: [PATCH 191/752] linting is super strict, ignoring the final unimportant ones --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 9ca9022370ea..dc4c386eb919 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -248,8 +248,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, costs = np.array(cost_list + virtual_costs) edges = np.array(edge_list + virtual_edges) else: - costs = cost_list - edges = edge_list + costs = cost_list # type: ignore + edges = edge_list # type: ignore vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) @@ -262,7 +262,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, new_virtual_edges = [mapping_n[i] for i in virtual_vertices] new_selected_edges = np.array(selected_edges + new_virtual_edges) else: - new_selected_edges = selected_edges + new_selected_edges = selected_edges # type: ignore edge_index = e_idx[:, new_selected_edges] new_selected_nodes = np.unique( From 54e1ad8f1909af1f2def4d341f381fb162386735 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 00:44:21 +0000 Subject: [PATCH 192/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index dc4c386eb919..79e5d1dadd1a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -248,8 +248,8 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, costs = np.array(cost_list + virtual_costs) edges = np.array(edge_list + virtual_edges) else: - costs = cost_list # type: ignore - edges = edge_list # type: ignore + costs = cost_list # type: ignore + edges = edge_list # type: ignore vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, pruning, verbosity_level) @@ -262,7 +262,7 @@ def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, new_virtual_edges = [mapping_n[i] for i in virtual_vertices] new_selected_edges = np.array(selected_edges + new_virtual_edges) else: - new_selected_edges = selected_edges # type: ignore + new_selected_edges = selected_edges # type: ignore edge_index = e_idx[:, new_selected_edges] new_selected_nodes = np.unique( From 6abd9f98b1803997230b08cd2ae31d329b759eb7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 15 Mar 2024 14:35:06 -0700 Subject: [PATCH 193/752] in linting, induced a bug, solving... --- torch_geometric/datasets/web_qsp_dataset.py | 227 ++++++++++---------- 1 file changed, 108 insertions(+), 119 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index dc4c386eb919..790c3e5f940a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -34,6 +34,113 @@ from torch_geometric.data import Data, InMemoryDataset +def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, + textual_nodes: df, + textual_edges: df, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: + # from original G-Retriever work + w# https://arxiv.org/abs/2402.07630 + c = 0.01 + if len(textual_nodes) == 0 or len(textual_edges) == 0: + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + graph = Data(x=graph.x, edge_index=graph.edge_index, + edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + return graph, desc + + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 + if topk > 0: + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) + topk = min(topk, graph.num_nodes) + _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) + + n_prizes = torch.zeros_like(n_prizes) + n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() + else: + n_prizes = torch.zeros(graph.num_nodes) + + if topk_e > 0: + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + topk_e = min(topk_e, e_prizes.unique().size(0)) + + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + e_prizes[e_prizes < topk_e_values[-1]] = 0.0 + last_topk_e_value = topk_e + for k in range(topk_e): + indices = e_prizes == topk_e_values[k] + value = min((topk_e - k) / sum(indices), last_topk_e_value - c) + e_prizes[indices] = value + last_topk_e_value = value + # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) + else: + e_prizes = torch.zeros(graph.num_edges) + + costs = [] + edges = [] + vritual_n_prizes = [] + virtual_edges = [] + virtual_costs = [] + mapping_n = {} + mapping_e = {} + for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + prize_e = e_prizes[i] + if prize_e <= cost_e: + mapping_e[len(edges)] = i + edges.append((src, dst)) + costs.append(cost_e - prize_e) + else: + virtual_node_id = graph.num_nodes + len(vritual_n_prizes) + mapping_n[virtual_node_id] = i + virtual_edges.append((src, virtual_node_id)) + virtual_edges.append((virtual_node_id, dst)) + virtual_costs.append(0) + virtual_costs.append(0) + vritual_n_prizes.append(prize_e - cost_e) + + prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) + num_edges = len(edges) + if len(virtual_costs) > 0: + costs = np.array(costs + virtual_costs) + edges = np.array(edges + virtual_edges) + + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) + + selected_nodes = vertices[vertices < graph.num_nodes] + selected_edges = [mapping_e[e] for e in edges if e < num_edges] + virtual_vertices = vertices[vertices >= graph.num_nodes] + if len(virtual_vertices) > 0: + virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges + virtual_edges) + + edge_index = graph.edge_index[:, selected_edges] + selected_nodes = np.unique( + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + + n = textual_nodes.iloc[selected_nodes] + e = textual_edges.iloc[selected_edges] + desc = n.to_csv(index=False) + "\n" + e.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + + mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + + x = graph.x[selected_nodes] + edge_attr = graph.edge_attr[selected_edges] + src = [mapping[i] for i in edge_index[0].tolist()] + dst = [mapping[i] for i in edge_index[1].tolist()] + edge_index = torch.LongTensor([src, dst]) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) + + return data, desc + + class Dataset(torch.utils.data.Dataset): def __init__(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> None: @@ -169,124 +276,6 @@ def __init__( super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - def retrieval_via_pcst(self, graph: Data, q_emb: torch.Tensor, - textual_nodes: df, textual_edges: df, topk: int = 3, - topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: - # from original G-Retriever work - # https://arxiv.org/abs/2402.07630 - c = 0.01 - # explicit casting for linting - num_nodes: int = graph.num_nodes # type: ignore - num_edges: int = graph.num_edges - e_idx: torch.Tensor = graph.edge_index # type: ignore - e_attr: torch.Tensor = graph.edge_attr # type: ignore - node_feat = torch.Tensor(graph.x) - if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv( - index=False) + "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - new_graph = Data(x=node_feat, edge_index=e_idx, edge_attr=e_attr, - num_nodes=num_nodes) - return new_graph, desc - - root = -1 # unrooted - num_clusters = 1 - pruning = "gw" - verbosity_level = 0 - if topk > 0: - n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, node_feat) - topk = min(topk, num_nodes) - _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) - - n_prizes = torch.zeros_like(n_prizes) - n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() - else: - n_prizes = torch.zeros(num_nodes) - - if topk_e > 0: - e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, e_attr) - topk_e = min(topk_e, e_prizes.unique().size(0)) - - topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, - largest=True) - e_prizes[e_prizes < topk_e_values[-1]] = 0.0 - last_topk_e_value = topk_e - for k in range(topk_e): - indices = e_prizes == topk_e_values[k] - value = min((topk_e - k) / sum(indices), last_topk_e_value - c) - e_prizes[indices] = value - last_topk_e_value = value - else: - e_prizes = torch.zeros(num_edges) - - cost_list = [] - edge_list = [] # type: ignore - virtual_n_prizes = [] # type: ignore - virtual_edges = [] - virtual_costs = [] - mapping_n = {} - mapping_e = {} - for i, (src, dst) in enumerate(e_idx.T.numpy()): - prize_e = e_prizes[i] - if prize_e <= cost_e: - mapping_e[len(edge_list)] = i - edge_list.append((src, dst)) - cost_list.append(cost_e - prize_e) - else: - virtual_node_id = num_nodes + len(virtual_n_prizes) - mapping_n[virtual_node_id] = i - virtual_edges.append((src, virtual_node_id)) - virtual_edges.append((virtual_node_id, dst)) - virtual_costs.append(0) - virtual_costs.append(0) - virtual_n_prizes.append(prize_e - cost_e) - - prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) - num_edges = len(edge_list) - if len(virtual_costs) > 0: - costs = np.array(cost_list + virtual_costs) - edges = np.array(edge_list + virtual_edges) - else: - costs = cost_list # type: ignore - edges = edge_list # type: ignore - - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, - pruning, verbosity_level) - - selected_nodes = vertices[vertices < num_nodes] - selected_edges = [mapping_e[e] for e in edges if e < num_edges] - virtual_vertices = vertices[vertices >= num_nodes] - if len(virtual_vertices) > 0: - virtual_vertices = vertices[vertices >= num_nodes] - new_virtual_edges = [mapping_n[i] for i in virtual_vertices] - new_selected_edges = np.array(selected_edges + new_virtual_edges) - else: - new_selected_edges = selected_edges # type: ignore - - edge_index = e_idx[:, new_selected_edges] - new_selected_nodes = np.unique( - np.concatenate( - [selected_nodes, edge_index[0].numpy(), - edge_index[1].numpy()])) - - n = textual_nodes.iloc[new_selected_nodes] - e = textual_edges.iloc[new_selected_edges] - desc = n.to_csv(index=False) + "\n" + e.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - - mapping = {n: i for i, n in enumerate(new_selected_nodes.tolist())} - - x = node_feat[new_selected_nodes] - edge_attr = e_attr[new_selected_edges] - src = [mapping[i] for i in edge_index[0].tolist()] - dst = [mapping[i] for i in edge_index[1].tolist()] - edge_index = torch.LongTensor([src, dst]) - data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(selected_nodes)) - - return data, desc - @property def raw_file_names(self) -> List[str]: return [] @@ -360,7 +349,7 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = self.retrieval_via_pcst( + psct_subgraph, desc = retrieval_via_pcst( raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) psct_subgraph["question"] = question From 7e54909081139cc173d4d1da6ae9e25b5d385c35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 21:38:54 +0000 Subject: [PATCH 194/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 790c3e5f940a..362d792acbf0 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -34,13 +34,11 @@ from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, - textual_nodes: df, - textual_edges: df, topk: int = 3, - topk_e: int = 3, +def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, + textual_edges: df, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work - w# https://arxiv.org/abs/2402.07630 + w # https://arxiv.org/abs/2402.07630 c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( @@ -349,9 +347,9 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = retrieval_via_pcst( - raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, - cost_e=0.5) + psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + nodes, edges, topk=3, + topk_e=5, cost_e=0.5) psct_subgraph["question"] = question psct_subgraph["label"] = label psct_subgraph["desc"] = desc From cdc7870f3ab3647d1f5cd93d807d3ae420106121 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 15 Mar 2024 14:40:10 -0700 Subject: [PATCH 195/752] in linting, induced a bug, solving... --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 790c3e5f940a..6a1921740a62 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -40,7 +40,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work - w# https://arxiv.org/abs/2402.07630 + # https://arxiv.org/abs/2402.07630 c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( From 9977075aa8afcf67131a6adddf0aa2cca933539a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 15 Mar 2024 14:48:49 -0700 Subject: [PATCH 196/752] reverted, did necessary cleanup, ignoring all other lint complaints since they are unreasonable and exist since the linter thinks all atributes of a graph can either be a value or None --- torch_geometric/datasets/web_qsp_dataset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 365be6e870f6..cfbaf200d085 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -36,7 +36,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, topk: int = 3, topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: + cost_e: float = 0.5) -> Tuple[Data, str]: # type: ignore # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 @@ -79,7 +79,7 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, costs = [] edges = [] - vritual_n_prizes = [] + virtual_n_prizes = [] virtual_edges = [] virtual_costs = [] mapping_n = {} @@ -91,15 +91,15 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, edges.append((src, dst)) costs.append(cost_e - prize_e) else: - virtual_node_id = graph.num_nodes + len(vritual_n_prizes) + virtual_node_id = graph.num_nodes + len(virtual_n_prizes) mapping_n[virtual_node_id] = i virtual_edges.append((src, virtual_node_id)) virtual_edges.append((virtual_node_id, dst)) virtual_costs.append(0) virtual_costs.append(0) - vritual_n_prizes.append(prize_e - cost_e) + virtual_n_prizes.append(prize_e - cost_e) - prizes = np.concatenate([n_prizes, np.array(vritual_n_prizes)]) + prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) num_edges = len(edges) if len(virtual_costs) > 0: costs = np.array(costs + virtual_costs) From 1f5d719e6cd5676088eb1703463df9f515d402cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 21:49:53 +0000 Subject: [PATCH 197/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index cfbaf200d085..2023c885c203 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -34,9 +34,10 @@ from torch_geometric.data import Data, InMemoryDataset -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, - textual_edges: df, topk: int = 3, topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: # type: ignore +def retrieval_via_pcst( + graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, + topk: int = 3, topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: # type: ignore # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 From 54d7008bc9ccc032bc6743fc1a0c85bad3d56bfb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 15 Mar 2024 14:58:32 -0700 Subject: [PATCH 198/752] reverted, did necessary cleanup, ignoring all other lint complaints since they are unreasonable and exist since the linter thinks all atributes of a graph can either be a value or None --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index cfbaf200d085..1e62014a6522 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, no_type_check import numpy as np @@ -33,10 +33,10 @@ from torch_geometric.data import Data, InMemoryDataset - +@no_type_check def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, topk: int = 3, topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: # type: ignore + cost_e: float = 0.5) -> Tuple[Data, str]: # from original G-Retriever work # https://arxiv.org/abs/2402.07630 c = 0.01 From 5c10425b8ec5c6d9594f406684380c5d832c338d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 22:00:35 +0000 Subject: [PATCH 199/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 1e62014a6522..2284321c2284 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -33,6 +33,7 @@ from torch_geometric.data import Data, InMemoryDataset + @no_type_check def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, topk: int = 3, topk_e: int = 3, From e0bedbbaa223eca392e594c58a0efc7f85e86571 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 26 Mar 2024 13:19:44 -0700 Subject: [PATCH 200/752] update to accomodate latest version of peft, int8->kbit --- examples/LLM_plus_GNN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/LLM_plus_GNN.py b/examples/LLM_plus_GNN.py index a2a62fd7b77e..e0f060c70e80 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/LLM_plus_GNN.py @@ -15,7 +15,7 @@ import pandas as pd import torch import torch.nn as nn -from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training +from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer @@ -134,7 +134,7 @@ def __init__(self, llm_model_path: str): **kwargs) print("Training LLAMA with LORA!") - self.model = prepare_model_for_int8_training(model) + self.model = prepare_model_for_kbit_training(model) lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 From 9c05795fb127c3c37f5040825062ad48c4bd9c15 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 1 Apr 2024 11:26:46 -0700 Subject: [PATCH 201/752] addresing Serge review --- examples/README.md | 8 +-- examples/llm_plus_gnn/README.md | 4 ++ .../g_retriever.py} | 56 ++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 examples/llm_plus_gnn/README.md rename examples/{LLM_plus_GNN.py => llm_plus_gnn/g_retriever.py} (90%) diff --git a/examples/README.md b/examples/README.md index 657bf8c10391..a31bf851dacf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,10 +14,10 @@ For examples on [Open Graph Benchmark](https://ogb.stanford.edu/) datasets, see - [`ogbn_papers_100m.py`](./ogbn_papers_100m.py) is an example for training a GNN on the large-scale `ogbn-papers100m` dataset, containing approximately ~1.6B edges. - [`ogbn_papers_100m_cugraph.py`](./ogbn_papers_100m_cugraph.py) shows how to accelerate the `ogbn-papers100m` workflow using [CuGraph](https://github.com/rapidsai/cugraph). -For an example on co-training a LLM with a GNN by using both textual and knowledge graph data, see [`examples/LLM_plus_GNN.py`](./LLM_plus_GNN.py). +For examples on co-training LLM with GNN, see examples and README under [`examples/llm_plus_gnn`](./llm_plus_gnn). -For examples on using `torch.compile`, see the examples under [`examples/compile`](./compile). +For examples on using `torch.compile`, see examples and README under [`examples/compile`](./compile). -For examples on scaling PyG up via multi-GPUs, see the examples under [`examples/multi_gpu`](./multi_gpu). +For examples on scaling PyG up via multi-GPUs, see examples and README under [`examples/multi_gpu`](./multi_gpu). -For examples on working with heterogeneous data, see the examples under [`examples/hetero`](./hetero). +For examples on working with heterogeneous data, see examples and README under [`examples/hetero`](./hetero). diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md new file mode 100644 index 000000000000..e923db16f6f2 --- /dev/null +++ b/examples/llm_plus_gnn/README.md @@ -0,0 +1,4 @@ +# Examples for LLM and GNN co-training +| Example | Description | +| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retriever Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | \ No newline at end of file diff --git a/examples/LLM_plus_GNN.py b/examples/llm_plus_gnn/g_retriever.py similarity index 90% rename from examples/LLM_plus_GNN.py rename to examples/llm_plus_gnn/g_retriever.py index e0f060c70e80..2de13cf58b5d 100644 --- a/examples/LLM_plus_GNN.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -6,7 +6,7 @@ requirements on top of basic PyG: pip install peft datasets transformers pcst_fast sentencepiece tqdm pandas """ - +import argparse import gc import math import re @@ -19,7 +19,6 @@ from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer - import torch_geometric from torch_geometric import seed_everything from torch_geometric.data import Batch, DataLoader @@ -30,21 +29,6 @@ EOS_USER = '[/INST]' EOS = '[/s]' IGNORE_INDEX = -100 -num_epochs = 10 - - -def adjust_learning_rate(param_group, LR, epoch): - # Decay the learning rate with half-cycle cosine after warmup - min_lr = 5e-6 - warmup_epochs = 1 - if epoch < warmup_epochs: - lr = LR - else: - lr = min_lr + (LR - min_lr) * 0.5 * ( - 1.0 + math.cos(math.pi * (epoch - warmup_epochs) / - (num_epochs - warmup_epochs))) - param_group["lr"] = lr - return lr def compute_accuracy(eval_output): @@ -92,7 +76,7 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): - def __init__(self, llm_model_path: str): + def __init__(self, llm_model_path: str, hidden_channels:int, num_gnn_layers: int): super().__init__() self.max_txt_len = 512 self.max_new_tokens = 32 @@ -156,8 +140,8 @@ def __init__(self, llm_model_path: str): self.graph_encoder = torch_geometric.nn.models.GAT( in_channels=1024, out_channels=1024, - hidden_channels=1024, - num_layers=4, + hidden_channels=hidden_channels, + num_layers=num_gnn_layers, heads=4, ).to(self.model.device) self.projector = nn.Sequential( @@ -333,7 +317,20 @@ def print_trainable_params(self): return trainable_params, all_param -def main(since: float): +def main(since: float, num_epochs: int, hidden_channels: int, num_gnn_layers: int, batch_size: int, lr: float): + def adjust_learning_rate(param_group, LR, epoch): + # Decay the learning rate with half-cycle cosine after warmup + min_lr = 5e-6 + warmup_epochs = 1 + if epoch < warmup_epochs: + lr = LR + else: + lr = min_lr + (LR - min_lr) * 0.5 * ( + 1.0 + math.cos(math.pi * (epoch - warmup_epochs) / + (num_epochs - warmup_epochs))) + param_group["lr"] = lr + return lr + seed_everything(42) dataset = WebQSPDataset() @@ -344,19 +341,18 @@ def main(since: float): val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] - train_loader = DataLoader(train_dataset, batch_size=4, drop_last=True, + train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, pin_memory=True, shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=4, drop_last=False, + val_loader = DataLoader(val_dataset, batch_size=batch_size, drop_last=False, pin_memory=True, shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=4, drop_last=False, + test_loader = DataLoader(test_dataset, batch_size=batch_size, drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf") + model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf", hidden_channels, num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] - lr = 1e-5 optimizer = torch.optim.AdamW([ { 'params': params, @@ -431,8 +427,14 @@ def main(since: float): if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--hidden_channels', type=int, default=1024) + parser.add_argument('--num_gnn_layers', type=int, default=4) + parser.add_argument('--lr', type=float, default=1e-5) + parser.add_argument('--epochs', type=int, default=10) + parser.add_argument('--batch_size', type=int, default=4) since = time.time() - prep_time = main(since) + prep_time = main(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 4c945a3c73c859f91b0ea092ae42f7b5211ed09e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:28:36 +0000 Subject: [PATCH 202/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/README.md | 7 ++++--- examples/llm_plus_gnn/g_retriever.py | 25 +++++++++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index e923db16f6f2..5c7900479540 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,4 +1,5 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retriever Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | \ No newline at end of file + +| Example | Description | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retriever Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2de13cf58b5d..505c0866373a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -19,6 +19,7 @@ from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer + import torch_geometric from torch_geometric import seed_everything from torch_geometric.data import Batch, DataLoader @@ -76,7 +77,8 @@ def compute_accuracy(eval_output): class GAT_LLAMA(nn.Module): - def __init__(self, llm_model_path: str, hidden_channels:int, num_gnn_layers: int): + def __init__(self, llm_model_path: str, hidden_channels: int, + num_gnn_layers: int): super().__init__() self.max_txt_len = 512 self.max_new_tokens = 32 @@ -317,7 +319,8 @@ def print_trainable_params(self): return trainable_params, all_param -def main(since: float, num_epochs: int, hidden_channels: int, num_gnn_layers: int, batch_size: int, lr: float): +def main(since: float, num_epochs: int, hidden_channels: int, + num_gnn_layers: int, batch_size: int, lr: float): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -341,15 +344,16 @@ def adjust_learning_rate(param_group, LR, epoch): val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] - train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, - pin_memory=True, shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=batch_size, drop_last=False, - pin_memory=True, shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=batch_size, drop_last=False, - pin_memory=True, shuffle=False) + train_loader = DataLoader(train_dataset, batch_size=batch_size, + drop_last=True, pin_memory=True, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=batch_size, + drop_last=False, pin_memory=True, shuffle=False) + test_loader = DataLoader(test_dataset, batch_size=batch_size, + drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf", hidden_channels, num_gnn_layers) + model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf", hidden_channels, + num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -434,7 +438,8 @@ def adjust_learning_rate(param_group, LR, epoch): parser.add_argument('--epochs', type=int, default=10) parser.add_argument('--batch_size', type=int, default=4) since = time.time() - prep_time = main(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) + prep_time = main(since, args.epochs, args.hidden_channels, + args.num_gnn_layers, args.batch_size, args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 1ee3fe97b4c34c59ae4d3e3471ab0379345736d8 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Apr 2024 11:22:53 -0700 Subject: [PATCH 203/752] minor cleanup --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 505c0866373a..4631aba29acc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -437,6 +437,7 @@ def adjust_learning_rate(param_group, LR, epoch): parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) parser.add_argument('--batch_size', type=int, default=4) + args = parser.parse_args() since = time.time() prep_time = main(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) From ff4ec907167cdd91d625ec4f6085ba10206d5fac Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 4 Apr 2024 14:37:38 -0700 Subject: [PATCH 204/752] adding a demo --- examples/llm_plus_gnn/g_retriever.py | 348 ++++++++++++++++++--------- 1 file changed, 238 insertions(+), 110 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 4631aba29acc..2bb430dbb754 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -11,7 +11,7 @@ import math import re import time - +from os import path import pandas as pd import torch import torch.nn as nn @@ -30,6 +30,20 @@ EOS_USER = '[/INST]' EOS = '[/s]' IGNORE_INDEX = -100 +llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" +max_txt_len = 512 +max_new_tokens = 32 +pad_token_id = 0 +padding_side = 'left' + +def detect_hallucinate(pred, label): + try: + pred = pred.split('[/s]')[0].strip().split('|') + correct_hit = len(re.findall(pred[0], label)) > 0 + hallucination = not correct_hit + return hallucination + except: # noqa + return "skip" def compute_accuracy(eval_output): @@ -76,51 +90,131 @@ def compute_accuracy(eval_output): return hit -class GAT_LLAMA(nn.Module): - def __init__(self, llm_model_path: str, hidden_channels: int, - num_gnn_layers: int): +def get_llm_kwargs(): + assert torch.cuda.is_available(), "GPU needed!" + avail_gpus = torch.cuda.device_count() + kwargs = { + "revision": "main", + } + max_mem_dict = {} + avail_mem_dict = {} + mem_total = 0 + gpus_2_use_4_llm = 0 + for i in range(avail_gpus): + available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024**3) + mem_total += available_mem + avail_mem_dict[i] = available_mem + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications + # >= 75 GB VRAM in total is recommended + if mem_total >= 75: + break + + for i in range(gpus_2_use_4_llm): + max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" + kwargs["max_memory"] = max_mem_dict + kwargs["device_map"] = "auto" + return kwargs + + +class LLAMA2(nn.Module): + # Pure LLAMA2 LLM module for demo + def __init__(self): super().__init__() - self.max_txt_len = 512 - self.max_new_tokens = 32 - print('Loading LLAMA') - assert torch.cuda.is_available(), "GPU needed!" - avail_gpus = torch.cuda.device_count() - kwargs = { - "revision": "main", - } - max_mem_dict = {} - avail_mem_dict = {} - mem_total = 0 - gpus_2_use_4_llm = 0 - for i in range(avail_gpus): - available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024**3) - mem_total += available_mem - avail_mem_dict[i] = available_mem - gpus_2_use_4_llm += 1 - # We want to use the minimum number of GPUs that LLM can fit on - # this is to minimize the need for interGPU communications - # >= 75 GB VRAM in total is recommended - if mem_total >= 75: - break - - for i in range(gpus_2_use_4_llm): - max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" - kwargs["max_memory"] = max_mem_dict - kwargs["device_map"] = "auto" + kwargs = get_llm_kwargs() print("Setting up LLAMA w/ kwargs =", kwargs) - llm_model_path = llm_model_path - self.tokenizer = AutoTokenizer.from_pretrained(llm_model_path, + self.tokenizer = AutoTokenizer.from_pretrained(llama2_str_name, use_fast=False) - self.tokenizer.pad_token_id = 0 - self.tokenizer.padding_side = 'left' - - model = AutoModelForCausalLM.from_pretrained( - llm_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, + self.tokenizer.pad_token_id = pad_token_id + self.tokenizer.padding_side = padding_side + self.llm = AutoModelForCausalLM.from_pretrained( + llama2_str_name, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, **kwargs) + self.llm_device = self.llm.device + self.word_embedding = self.llm.model.get_input_embeddings() + + + def encode_inputs(self, samples: Batch): + batch_size = len(samples['question']) + questions = self.tokenizer(samples["question"], + add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], + add_special_tokens=False) + + # encode special tokens + eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) + bos_embeds = self.word_embedding( + self.tokenizer(BOS, add_special_tokens=False, + return_tensors='pt').input_ids[0].to( + self.llm_device)) + pad_embeds = self.word_embedding( + torch.tensor(self.tokenizer.pad_token_id).to( + self.llm_device)).unsqueeze(0) + return batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds + + def inference(self, samples: Batch): + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs(samples) + batch_inputs_embeds = [] + batch_attention_mask = [] + for i in range(batch_size): + # Add bos & eos token + input_ids = descriptions.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + inputs_embeds = torch.cat( + [bos_embeds, inputs_embeds], + dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + outputs = self.llm.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_new_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return { + 'pred': pred, + 'label': samples['label'], + 'question': samples['question'], + 'desc': samples['desc'], + } + + +class GAT_LLAMA(nn.Module): + def __init__(self, hidden_channels: int, num_gnn_layers: int): + super().__init__() + + + self.llama2 = LLAMA2() print("Training LLAMA with LORA!") - self.model = prepare_model_for_kbit_training(model) + self.llm = self.llama2.llm + self.llm_device = self.llama2.llm_device + self.llm = prepare_model_for_kbit_training(self.llm) + self.tokenizer = self.llama2.tokenizer lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 @@ -136,7 +230,8 @@ def __init__(self, llm_model_path: str, hidden_channels: int, bias="none", task_type="CAUSAL_LM", ) - self.model = get_peft_model(self.model, config) + self.llm = get_peft_model(self.llm, config) + print('Finish loading LLAMA!') self.graph_encoder = torch_geometric.nn.models.GAT( @@ -145,42 +240,31 @@ def __init__(self, llm_model_path: str, hidden_channels: int, hidden_channels=hidden_channels, num_layers=num_gnn_layers, heads=4, - ).to(self.model.device) + ).to(self.llm_device) self.projector = nn.Sequential( nn.Linear(1024, 2048), nn.Sigmoid(), nn.Linear(2048, 4096), - ).to(self.model.device) + ).to(self.llm_device) - self.word_embedding = self.model.model.get_input_embeddings() + self.word_embedding = self.llama2.word_embedding def encode_graphs(self, samples: Batch): - x = samples.x.to(self.model.device) - edge_index = samples.edge_index.long().to(self.model.device) - edge_attr = samples.edge_attr.to(self.model.device) + x = samples.x.to(self.llm_device) + edge_index = samples.edge_index.long().to(self.llm_device) + edge_attr = samples.edge_attr.to(self.llm_device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - batch = samples.batch.to(self.model.device) + batch = samples.batch.to(self.llm_device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds def forward(self, samples: Batch): - # encode description, questions and labels - batch_size = len(samples.question) - questions = self.tokenizer(samples.question, add_special_tokens=False) - descriptions = self.tokenizer(samples.desc, add_special_tokens=False) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) + # encode labels labels = self.tokenizer(samples.label, add_special_tokens=False) - - # encode special tokens + # encode training specific special token eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding( - self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to( - self.model.device)) - pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to( - self.model.device)).unsqueeze(0) # encode graphs graph_embeds = self.encode_graphs(samples) @@ -192,12 +276,12 @@ def forward(self, samples: Batch): for i in range(batch_size): # Add bos & eos token label_input_ids = labels.input_ids[ - i][:self.max_new_tokens] + eos_tokens.input_ids + i][:max_new_tokens] + eos_tokens.input_ids input_ids = descriptions.input_ids[ - i][:self.max_txt_len] + questions.input_ids[ + i][:max_txt_len] + questions.input_ids[ i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.model.device)) + torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) @@ -222,40 +306,23 @@ def forward(self, samples: Batch): i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.model.device) + dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to( - self.model.device) + self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( - self.model.device) + self.llm_device) with torch.cuda.amp.autocast(dtype=torch.bfloat16): - outputs = self.model( + outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, return_dict=True, labels=label_input_ids, ) - return outputs.loss def inference(self, samples: Batch): - # encode description and questions - batch_size = len(samples['question']) - questions = self.tokenizer(samples["question"], - add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], - add_special_tokens=False) - - # encode special tokens - eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding( - self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to( - self.model.device)) - pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to( - self.model.device)).unsqueeze(0) - + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) # encode graphs graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) @@ -265,10 +332,10 @@ def inference(self, samples: Batch): for i in range(batch_size): # Add bos & eos token input_ids = descriptions.input_ids[ - i][:self.max_txt_len] + questions.input_ids[ + i][:max_txt_len] + questions.input_ids[ i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.model.device)) + torch.tensor(input_ids).to(self.llm_device)) inputs_embeds = torch.cat( [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], dim=0) @@ -285,14 +352,14 @@ def inference(self, samples: Batch): ] * pad_length + batch_attention_mask[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.model.device) + dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to( - self.model.device) + self.llm_device) with torch.cuda.amp.autocast(dtype=torch.bfloat16): - outputs = self.model.generate( + outputs = self.llm.generate( inputs_embeds=inputs_embeds, - max_new_tokens=self.max_new_tokens, + max_new_tokens=max_new_tokens, attention_mask=attention_mask, # do_sample=True, use_cache=True # IMPORTANT! @@ -352,8 +419,7 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GAT_LLAMA("meta-llama/Llama-2-7b-chat-hf", hidden_channels, - num_gnn_layers) + model = GAT_LLAMA(hidden_channels, num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -427,23 +493,85 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 6 Post-processing & compute metrics acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') - return prep_time + # save model + torch.save(model, "gat_llama.pt") + return prep_time, dataset, model +def minimal_demo(model, dataset): + # Step 1: Define a single batch size test loader + idx_split = dataset.split_idxs + test_dataset = [dataset[i] for i in idx_split['test']] + # batch size 1 loader for simplicity + loader = DataLoader(test_dataset, batch_size=1, drop_last=False, + pin_memory=True, shuffle=False) + # define the pure pretrained LLM + pure_llm = LLAMA2() + + # Step loop through the loader and run both models + gnn_llm_hallucin_sum = 0 + pure_llm_hallucin_sum = 0 + final_prnt_str = "" + print("Checking LLM vs GNN+LLM for hallucinations...") + for batch in tqdm(loader): + question = batch.question[0] + correct_answer = batch.label[0] + gnn_llm_out = model.inference(batch) + pure_llm_out = pure_llm.inference(batch) + gnn_llm_pred = gnn_llm_out['pred'][0] + pure_llm_pred = pure_llm_out['pred'][0] + gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) + pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": + # skipping since hard to evaluate if the answer's are hallucinations + continue + gnn_llm_hallucin_sum += bool(gnn_llm_hallucinates) + pure_llm_hallucin_sum += bool(pure_llm_hallucinates) + # showcase LLM hallucinations solved by GNN + if pure_llm_hallucinates and not gnn_llm_hallucinates: + final_prnt_str += "Question: " + question + "\n" + final_prnt_str += "Correct Answer: " + correct_answer + "\n" + final_prnt_str += "Pure LLM Prediction: " + pure_llm_pred + "\n" + final_prnt_str += "GNN+LLM Prediction:" + gnn_llm_pred + "\n" + final_prnt_str += "#"*20 + "\n" + print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) + print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + percent = 100.0 * round(1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum) , 2) + print(f"GNN reduces hallucinations by: ~{percent}%") + print("Note: hallucinations detected by regex hence the ~") + print("Instances where GNN solves the hallucinations of Pure LLMs:") + print(final_prnt_str) if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--hidden_channels', type=int, default=1024) - parser.add_argument('--num_gnn_layers', type=int, default=4) - parser.add_argument('--lr', type=float, default=1e-5) - parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=4) - args = parser.parse_args() - since = time.time() - prep_time = main(since, args.epochs, args.hidden_channels, + # check if saved model + if path.exists("gat_llama.pt"): + print("Existing trained model found.") + # ask if want to retrain or skip to demo + print("Would you like to retrain?") + user_input = str(input("(y/n):")).lower() + retrain = user_input == "y" + else: + retrain = True + if retrain: + parser = argparse.ArgumentParser() + parser.add_argument('--hidden_channels', type=int, default=1024) + parser.add_argument('--num_gnn_layers', type=int, default=4) + parser.add_argument('--lr', type=float, default=1e-5) + parser.add_argument('--epochs', type=int, default=10) + parser.add_argument('--batch_size', type=int, default=4) + args = parser.parse_args() + since = time.time() + prep_time, dataset, model = main(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) - torch.cuda.empty_cache() - torch.cuda.reset_max_memory_allocated() - gc.collect() - e2e_time = round(time.time() - since, 2) - print("E2E time (e2e_time) =", e2e_time) - print("E2E time minus Prep Time =", e2e_time - prep_time) + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() + e2e_time = round(time.time() - since, 2) + print("E2E time (e2e_time) =", e2e_time) + print("E2E time minus Prep Time =", e2e_time - prep_time) + else: + model = torch.load("gat_llama.pt") + dataset = WebQSPDataset() + print("Would you like a minimal demo showcasing how GNN+LLM can solve LLM hallucinations?") + user_input = str(input("(y/n):")).lower() + if user_input == "y": + minimal_demo(model, dataset) From e60b280fbe570d0e98d47b3ac72f4edc91f8e806 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 4 Apr 2024 15:36:10 -0700 Subject: [PATCH 205/752] beginning drafting, very rough --- torch_geometric/nn/models/gnn_llm.py | 286 +++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 torch_geometric/nn/models/gnn_llm.py diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py new file mode 100644 index 000000000000..0f355697eff5 --- /dev/null +++ b/torch_geometric/nn/models/gnn_llm.py @@ -0,0 +1,286 @@ +import torch +import torch.nn as nn +from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training +from transformers import AutoModelForCausalLM, AutoTokenizer + +import torch_geometric +from torch_geometric import seed_everything +from torch_geometric.data import Batch +from torch_geometric.utils import scatter +from torch_geometric.nn.models import GAT + +BOS = '[INST]' +EOS_USER = '[/INST]' +EOS = '[/s]' +IGNORE_INDEX = -100 +llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" +max_txt_len = 512 +max_new_tokens = 32 +pad_token_id = 0 +padding_side = 'left' + +class LLM(nn.Module): + def __init__(self): + super().__init__() + print('Loading LLAMA') + kwargs = get_llm_kwargs() + print("Setting up LLAMA w/ kwargs =", kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(llama2_str_name, + use_fast=False) + self.tokenizer.pad_token_id = pad_token_id + self.tokenizer.padding_side = padding_side + self.llm = AutoModelForCausalLM.from_pretrained( + llama2_str_name, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, + **kwargs) + self.llm_device = self.llm.device + self.word_embedding = self.llm.model.get_input_embeddings() + + + def encode_inputs(self, samples: Batch): + batch_size = len(samples['question']) + questions = self.tokenizer(samples["question"], + add_special_tokens=False) + descriptions = self.tokenizer(samples["desc"], + add_special_tokens=False) + + # encode special tokens + eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) + bos_embeds = self.word_embedding( + self.tokenizer(BOS, add_special_tokens=False, + return_tensors='pt').input_ids[0].to( + self.llm_device)) + pad_embeds = self.word_embedding( + torch.tensor(self.tokenizer.pad_token_id).to( + self.llm_device)).unsqueeze(0) + return batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds + + def inference(self, samples: Batch): + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs(samples) + batch_inputs_embeds = [] + batch_attention_mask = [] + for i in range(batch_size): + # Add bos & eos token + input_ids = descriptions.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + inputs_embeds = torch.cat( + [bos_embeds, inputs_embeds], + dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + outputs = self.llm.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_new_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return { + 'pred': pred, + 'label': samples['label'], + 'question': samples['question'], + 'desc': samples['desc'], + } + + +class GNN_LLM(nn.Module): + def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, hidden_channels: int, num_gnn_layers: int): + super().__init__() + + + self.llama2 = LLAMA2() + + print("Training LLAMA with LORA!") + self.llm = self.llama2.llm + self.llm_device = self.llama2.llm_device + self.llm = prepare_model_for_kbit_training(self.llm) + self.tokenizer = self.llama2.tokenizer + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + self.llm = get_peft_model(self.llm, config) + + print('Finish loading LLAMA!') + + self.graph_encoder = gnn_to_use( + in_channels=1024, + out_channels=1024, + hidden_channels=hidden_channels, + num_layers=num_gnn_layers, + heads=4, + ).to(self.llm_device) + self.projector = nn.Sequential( + nn.Linear(1024, 2048), + nn.Sigmoid(), + nn.Linear(2048, 4096), + ).to(self.llm_device) + + self.word_embedding = self.llama2.word_embedding + + def encode_graphs(self, samples: Batch): + x = samples.x.to(self.llm_device) + edge_index = samples.edge_index.long().to(self.llm_device) + edge_attr = samples.edge_attr.to(self.llm_device) + n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) + batch = samples.batch.to(self.llm_device) + # mean pooling + g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') + return g_embeds + + def forward(self, samples: Batch): + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) + # encode labels + labels = self.tokenizer(samples.label, add_special_tokens=False) + # encode training specific special token + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + + # encode graphs + graph_embeds = self.encode_graphs(samples) + graph_embeds = self.projector(graph_embeds) + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[ + i][:max_new_tokens] + eos_tokens.input_ids + input_ids = descriptions.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.llm_device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + outputs = self.llm( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + return outputs.loss + + def inference(self, samples: Batch): + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) + # encode graphs + graph_embeds = self.encode_graphs(samples) + graph_embeds = self.projector(graph_embeds) + + batch_inputs_embeds = [] + batch_attention_mask = [] + for i in range(batch_size): + # Add bos & eos token + input_ids = descriptions.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + inputs_embeds = torch.cat( + [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], + dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + outputs = self.llm.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_new_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return { + 'pred': pred, + 'label': samples['label'], + 'question': samples['question'], + 'desc': samples['desc'], + } + + def print_trainable_params(self): + trainable_params = 0 + all_param = 0 + for _, param in self.named_parameters(): + num_params = param.numel() + + all_param += num_params + if param.requires_grad: + trainable_params += num_params + + return trainable_params, all_param From 03280354461693f7e5c20d02890de7260503d745 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 4 Apr 2024 16:25:02 -0700 Subject: [PATCH 206/752] drafting wip --- torch_geometric/nn/models/gnn_llm.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 0f355697eff5..94e189920efb 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -7,7 +7,8 @@ from torch_geometric import seed_everything from torch_geometric.data import Batch from torch_geometric.utils import scatter -from torch_geometric.nn.models import GAT +from torch_geometric.nn.models import GAT, BasicGNN + BOS = '[INST]' EOS_USER = '[/INST]' @@ -104,12 +105,27 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): + """This GNN+LLM implementation is based on the design from + G-retriever. Original Paper: https://arxiv.org/abs/2402.07630 + Args: + llm_to_use (str): A string representing the huggingface model you + want to use. This module has been tested for 'llama2' and 'gemma2'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on https://github.com/pyg-team/pytorch_geometric + and tag puririshi98. (default: :obj:'llama2') + gnn_to_use (BasicGNN): Please pass a valid model that extends + torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) + """ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, hidden_channels: int, num_gnn_layers: int): super().__init__() - - self.llama2 = LLAMA2() - + if 'llama' in llm_to_use.lower(): + self.llm_to_use = LLM('llama2') + elif 'gemma' in llm_to_use.lower(): + self.llm_to_use = LLM('gemma2') + else: + self.llm_to_use = LLM(llm_to_use) print("Training LLAMA with LORA!") self.llm = self.llama2.llm self.llm_device = self.llama2.llm_device From 9988e4e3cb22b5ed723a96089702a5d577b75e04 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 09:00:54 -0700 Subject: [PATCH 207/752] drafting wip --- torch_geometric/nn/models/gnn_llm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 94e189920efb..be5ba0bb5819 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -117,7 +117,7 @@ class GNN_LLM(nn.Module): gnn_to_use (BasicGNN): Please pass a valid model that extends torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) """ - def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, hidden_channels: int, num_gnn_layers: int): + def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden_channels: int, out_channels: int, num_gnn_layers: int, num_gnn_heads: int = 4): super().__init__() if 'llama' in llm_to_use.lower(): @@ -151,11 +151,11 @@ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, hidden_channels: int, nu print('Finish loading LLAMA!') self.graph_encoder = gnn_to_use( - in_channels=1024, - out_channels=1024, + in_channels=in_channels, + out_channels=out_channels, hidden_channels=hidden_channels, num_layers=num_gnn_layers, - heads=4, + heads=num_gnn_heads, ).to(self.llm_device) self.projector = nn.Sequential( nn.Linear(1024, 2048), From 497d3c0b1c5fea2cef5ea29d039194ddcac0cd97 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 09:13:12 -0700 Subject: [PATCH 208/752] drafting wip --- torch_geometric/nn/models/gnn_llm.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index be5ba0bb5819..3e0a585f1dff 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -15,6 +15,7 @@ EOS = '[/s]' IGNORE_INDEX = -100 llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" +gemma2_str_name = # max_txt_len = 512 max_new_tokens = 32 pad_token_id = 0 @@ -117,9 +118,10 @@ class GNN_LLM(nn.Module): gnn_to_use (BasicGNN): Please pass a valid model that extends torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) """ - def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden_channels: int, out_channels: int, num_gnn_layers: int, num_gnn_heads: int = 4): + def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, + in_channels: int, hidden_channels: int, out_channels: int, + num_gnn_layers: int, num_gnn_heads: int = 4): super().__init__() - if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2') elif 'gemma' in llm_to_use.lower(): @@ -127,10 +129,10 @@ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden else: self.llm_to_use = LLM(llm_to_use) print("Training LLAMA with LORA!") - self.llm = self.llama2.llm - self.llm_device = self.llama2.llm_device + self.llm = self.llm_to_use.llm + self.llm_device = self.llm_to_use.llm_device self.llm = prepare_model_for_kbit_training(self.llm) - self.tokenizer = self.llama2.tokenizer + self.tokenizer = self.llm_to_use.tokenizer lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 @@ -163,7 +165,7 @@ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden nn.Linear(2048, 4096), ).to(self.llm_device) - self.word_embedding = self.llama2.word_embedding + self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, samples: Batch): x = samples.x.to(self.llm_device) @@ -176,7 +178,7 @@ def encode_graphs(self, samples: Batch): return g_embeds def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) # encode labels labels = self.tokenizer(samples.label, add_special_tokens=False) # encode training specific special token @@ -238,7 +240,7 @@ def forward(self, samples: Batch): return outputs.loss def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) # encode graphs graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) From e509faa7004d79e6ec794d053c6dc42f7f0cbadb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 09:52:42 -0700 Subject: [PATCH 209/752] drafting wip --- torch_geometric/nn/models/__init__.py | 1 + torch_geometric/nn/models/gnn_llm.py | 55 +++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index 5494311e902d..9b70b8b73875 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -74,4 +74,5 @@ 'PMLP', 'NeuralFingerprint', 'ViSNet', + 'GNN_LLM' ] diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 3e0a585f1dff..bfe7ff61621e 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -15,18 +15,54 @@ EOS = '[/s]' IGNORE_INDEX = -100 llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" -gemma2_str_name = # +gemma_str_name = "google/gemma-7b" max_txt_len = 512 max_new_tokens = 32 pad_token_id = 0 padding_side = 'left' +def get_llm_kwargs(mem_needed): + assert torch.cuda.is_available(), "GPU needed!" + avail_gpus = torch.cuda.device_count() + kwargs = { + "revision": "main", + } + max_mem_dict = {} + avail_mem_dict = {} + mem_total = 0 + gpus_2_use_4_llm = 0 + for i in range(avail_gpus): + available_mem = int(torch.cuda.mem_get_info(i)[0] // 1024**3) + mem_total += available_mem + avail_mem_dict[i] = available_mem + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications + if mem_total >= mem_needed: + break + + for i in range(gpus_2_use_4_llm): + max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" + kwargs["max_memory"] = max_mem_dict + kwargs["device_map"] = "auto" + return kwargs + class LLM(nn.Module): - def __init__(self): + def __init__(self, llm_name: str, num_params: int = 7): super().__init__() - print('Loading LLAMA') - kwargs = get_llm_kwargs() - print("Setting up LLAMA w/ kwargs =", kwargs) + if llm_name == "llama2": + self.printable_llm_name = "LLAMA2" + self.huggingface_str = llama2_str_name + elif llm_name == "gemma": + self.printable_llm_name = "GEMMA" + self.huggingface_str = gemma_str_name + else: + self.printable_llm_name = llm_name + self.huggingface_str = llm_name + self.mem_needed = 75 * num_params / 7 + print('Loading ' + str(self.printable_llm_name)) + kwargs = get_llm_kwargs(self.mem_needed) + print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) self.tokenizer = AutoTokenizer.from_pretrained(llama2_str_name, use_fast=False) self.tokenizer.pad_token_id = pad_token_id @@ -110,22 +146,25 @@ class GNN_LLM(nn.Module): G-retriever. Original Paper: https://arxiv.org/abs/2402.07630 Args: llm_to_use (str): A string representing the huggingface model you - want to use. This module has been tested for 'llama2' and 'gemma2'. + want to use. This module has been tested for 'llama2' and 'gemma'. Other huggingface transformer models should work if you pass the correct name, see huggingface.co for details. If any issues occur please file an issue on https://github.com/pyg-team/pytorch_geometric and tag puririshi98. (default: :obj:'llama2') gnn_to_use (BasicGNN): Please pass a valid model that extends torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) + ... + num_llm_params (int): An integer representing how many params your + huggingface transformer model has, in billions. (default :obj:`7`) """ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden_channels: int, out_channels: int, - num_gnn_layers: int, num_gnn_heads: int = 4): + num_gnn_layers: int, num_gnn_heads: int = 4, num_llm_params: int = 7): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2') elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma2') + self.llm_to_use = LLM('gemma') else: self.llm_to_use = LLM(llm_to_use) print("Training LLAMA with LORA!") From d2873fcc41539c19838308c1f2093bb174f39eb9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 11:11:32 -0700 Subject: [PATCH 210/752] drafting wip --- torch_geometric/nn/models/gnn_llm.py | 69 ++++++++++++++++++---------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index bfe7ff61621e..7486ecbd50c9 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -22,7 +22,7 @@ padding_side = 'left' def get_llm_kwargs(mem_needed): - assert torch.cuda.is_available(), "GPU needed!" + assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" avail_gpus = torch.cuda.device_count() kwargs = { "revision": "main", @@ -153,13 +153,34 @@ class GNN_LLM(nn.Module): and tag puririshi98. (default: :obj:'llama2') gnn_to_use (BasicGNN): Please pass a valid model that extends torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) - ... + in_channels (int or tuple): Size of each input sample, or :obj:`-1` to + derive the size from the first input(s) to the forward method. + A tuple corresponds to the sizes of source and target + dimensionalities. + hidden_channels (int): Size of each hidden sample. + out_channels (int, optional): If not set to :obj:`None`, will apply a + final linear transformation to convert hidden node embeddings to + output size :obj:`out_channels`. (default: :obj:`None`) + num_gnn_layers (int): Number of message passing layers. + num_gnn_heads (int): Number of heads to use for BasicGNNs with the + `heads` kwarg. (default: 4) num_llm_params (int): An integer representing how many params your - huggingface transformer model has, in billions. (default :obj:`7`) + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the available + GPU memory of your GPUs (default :obj:`7`) + use_lora (bool): use LORA from peft for training the LLM. see + https://huggingface.co/docs/peft/en/index for details. + autocast_llm (bool): Wether to to use `torch.cuda.amp.autocast` + on the LLM or not. See https://pytorch.org/docs/stable/amp.html for + details. (default :obj:`True`) + llm_autocast_dtype (torch.dtype): The lower precision dtype to autocast + to. (default :obj: `torch.bloat16`) """ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, in_channels: int, hidden_channels: int, out_channels: int, - num_gnn_layers: int, num_gnn_heads: int = 4, num_llm_params: int = 7): + num_gnn_layers: int, num_gnn_heads: int = 4, num_llm_params: int = 7, + use_lora: bool = True, autocast_llm=True, + llm_autocast_dtype=torch.bfloat16): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2') @@ -167,29 +188,29 @@ def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, self.llm_to_use = LLM('gemma') else: self.llm_to_use = LLM(llm_to_use) - print("Training LLAMA with LORA!") self.llm = self.llm_to_use.llm + if use_lora: + print("Training our LLM with LORA!") + self.llm = prepare_model_for_kbit_training(self.llm) + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + self.llm = get_peft_model(self.llm, config) self.llm_device = self.llm_to_use.llm_device - self.llm = prepare_model_for_kbit_training(self.llm) self.tokenizer = self.llm_to_use.tokenizer - lora_r: int = 8 - lora_alpha: int = 16 - lora_dropout: float = 0.05 - lora_target_modules = [ - "q_proj", - "v_proj", - ] - config = LoraConfig( - r=lora_r, - lora_alpha=lora_alpha, - target_modules=lora_target_modules, - lora_dropout=lora_dropout, - bias="none", - task_type="CAUSAL_LM", - ) - self.llm = get_peft_model(self.llm, config) - - print('Finish loading LLAMA!') + print('Finished loading LLAMA!') self.graph_encoder = gnn_to_use( in_channels=in_channels, From 5426feb7413d4578d7e491966cf126cb56293ed2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 11:43:39 -0700 Subject: [PATCH 211/752] drafting wip --- torch_geometric/nn/models/gnn_llm.py | 79 +++++++++++++--------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 7486ecbd50c9..7609ea0c6774 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -48,7 +48,7 @@ def get_llm_kwargs(mem_needed): return kwargs class LLM(nn.Module): - def __init__(self, llm_name: str, num_params: int = 7): + def __init__(self, llm_name: str, num_params: int = 7, llm_dtype=torch.bfloat16): super().__init__() if llm_name == "llama2": self.printable_llm_name = "LLAMA2" @@ -60,15 +60,16 @@ def __init__(self, llm_name: str, num_params: int = 7): self.printable_llm_name = llm_name self.huggingface_str = llm_name self.mem_needed = 75 * num_params / 7 + self.llm_dtype = llm_dtype print('Loading ' + str(self.printable_llm_name)) kwargs = get_llm_kwargs(self.mem_needed) print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) - self.tokenizer = AutoTokenizer.from_pretrained(llama2_str_name, + self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, use_fast=False) self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( - llama2_str_name, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, + self.huggingface_str, torch_dtype=self.llm_dtype, low_cpu_mem_usage=True, **kwargs) self.llm_device = self.llm.device self.word_embedding = self.llm.model.get_input_embeddings() @@ -93,6 +94,7 @@ def encode_inputs(self, samples: Batch): return batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds def inference(self, samples: Batch): + # this function is for comparing a pretrained LLM to a trained GNN_LLM batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs(samples) batch_inputs_embeds = [] batch_attention_mask = [] @@ -123,7 +125,7 @@ def inference(self, samples: Batch): attention_mask = torch.tensor(batch_attention_mask).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=torch.bfloat16): + with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm.generate( inputs_embeds=inputs_embeds, max_new_tokens=max_new_tokens, @@ -143,52 +145,47 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): """This GNN+LLM implementation is based on the design from - G-retriever. Original Paper: https://arxiv.org/abs/2402.07630 + G-retriever. See `examples/llm_plus_gnn/g_retriever.py` + for an example. Original Paper: https://arxiv.org/abs/2402.07630 Args: llm_to_use (str): A string representing the huggingface model you - want to use. This module has been tested for 'llama2' and 'gemma'. - Other huggingface transformer models should work if you pass the - correct name, see huggingface.co for details. If any issues occur - please file an issue on https://github.com/pyg-team/pytorch_geometric - and tag puririshi98. (default: :obj:'llama2') + want to use. This module has been tested for 'llama2' and 'gemma'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on + https://github.com/pyg-team/pytorch_geometric + and assign to puririshi98. (default: :obj:'llama2') + use_lora (bool): use LORA from peft for training the LLM. see + https://huggingface.co/docs/peft/en/index for details. + llm_dtype (torch.dtype): The lower precision dtype to use for the + LLM. (default :obj: `torch.bloat16`) + num_llm_params (int): An integer representing how many params your + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the + available GPU memory of your GPUs (default :obj:`7`) gnn_to_use (BasicGNN): Please pass a valid model that extends - torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) - in_channels (int or tuple): Size of each input sample, or :obj:`-1` to - derive the size from the first input(s) to the forward method. - A tuple corresponds to the sizes of source and target - dimensionalities. - hidden_channels (int): Size of each hidden sample. - out_channels (int, optional): If not set to :obj:`None`, will apply a - final linear transformation to convert hidden node embeddings to - output size :obj:`out_channels`. (default: :obj:`None`) - num_gnn_layers (int): Number of message passing layers. + torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) + gnn_in_channels (int): (default: 1024) + gnn_hidden_channels (int): (default: 1024) + gnn_out_channels (int): (default: 1024) + num_gnn_layers (int): (default: 4) num_gnn_heads (int): Number of heads to use for BasicGNNs with the `heads` kwarg. (default: 4) - num_llm_params (int): An integer representing how many params your - huggingface transformer model has, in billions. This is used to - automatically allocate the number of gpus needed, given the available - GPU memory of your GPUs (default :obj:`7`) - use_lora (bool): use LORA from peft for training the LLM. see - https://huggingface.co/docs/peft/en/index for details. - autocast_llm (bool): Wether to to use `torch.cuda.amp.autocast` - on the LLM or not. See https://pytorch.org/docs/stable/amp.html for - details. (default :obj:`True`) - llm_autocast_dtype (torch.dtype): The lower precision dtype to autocast - to. (default :obj: `torch.bloat16`) """ - def __init__(self, llm_to_use='llama2', gnn_to_use=GAT, - in_channels: int, hidden_channels: int, out_channels: int, - num_gnn_layers: int, num_gnn_heads: int = 4, num_llm_params: int = 7, - use_lora: bool = True, autocast_llm=True, - llm_autocast_dtype=torch.bfloat16): + def __init__(self, llm_to_use='llama2', use_lora_for_llm: bool = True, + llm_dtype=torch.bfloat16, num_llm_params: int = 7, + gnn_to_use=GAT, gnn_in_channels: int = 1024, + gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, + num_gnn_layers: int = 4, num_gnn_heads: int = 4,): super().__init__() if 'llama' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2') + self.llm_to_use = LLM('llama2', llm_dtype) elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma') + self.llm_to_use = LLM('gemma', llm_dtype) else: - self.llm_to_use = LLM(llm_to_use) + self.llm_to_use = LLM(llm_to_use, llm_dtype) self.llm = self.llm_to_use.llm + self.llm_dtype = self.llm_dtype if use_lora: print("Training our LLM with LORA!") self.llm = prepare_model_for_kbit_training(self.llm) @@ -290,7 +287,7 @@ def forward(self, samples: Batch): label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=torch.bfloat16): + with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, @@ -334,7 +331,7 @@ def inference(self, samples: Batch): attention_mask = torch.tensor(batch_attention_mask).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=torch.bfloat16): + with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm.generate( inputs_embeds=inputs_embeds, max_new_tokens=max_new_tokens, From 2256578355d713ab09007b23b1bcb3da4af47bf2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 12:58:44 -0700 Subject: [PATCH 212/752] drafting wip --- examples/llm_plus_gnn/g_retriever.py | 303 +-------------------------- torch_geometric/nn/models/gnn_llm.py | 2 +- 2 files changed, 5 insertions(+), 300 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2bb430dbb754..fb246a34214e 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -25,6 +25,8 @@ from torch_geometric.data import Batch, DataLoader from torch_geometric.datasets import WebQSPDataset from torch_geometric.utils import scatter +from torch_geometric.nn.models.gnn_llm import LLM, GNN_LLM + BOS = '[INST]' EOS_USER = '[/INST]' @@ -89,303 +91,6 @@ def compute_accuracy(eval_output): return hit - -def get_llm_kwargs(): - assert torch.cuda.is_available(), "GPU needed!" - avail_gpus = torch.cuda.device_count() - kwargs = { - "revision": "main", - } - max_mem_dict = {} - avail_mem_dict = {} - mem_total = 0 - gpus_2_use_4_llm = 0 - for i in range(avail_gpus): - available_mem = int(torch.cuda.mem_get_info(0)[0] // 1024**3) - mem_total += available_mem - avail_mem_dict[i] = available_mem - gpus_2_use_4_llm += 1 - # We want to use the minimum number of GPUs that LLM can fit on - # this is to minimize the need for interGPU communications - # >= 75 GB VRAM in total is recommended - if mem_total >= 75: - break - - for i in range(gpus_2_use_4_llm): - max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" - kwargs["max_memory"] = max_mem_dict - kwargs["device_map"] = "auto" - return kwargs - - -class LLAMA2(nn.Module): - # Pure LLAMA2 LLM module for demo - def __init__(self): - super().__init__() - print('Loading LLAMA') - kwargs = get_llm_kwargs() - print("Setting up LLAMA w/ kwargs =", kwargs) - self.tokenizer = AutoTokenizer.from_pretrained(llama2_str_name, - use_fast=False) - self.tokenizer.pad_token_id = pad_token_id - self.tokenizer.padding_side = padding_side - self.llm = AutoModelForCausalLM.from_pretrained( - llama2_str_name, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, - **kwargs) - self.llm_device = self.llm.device - self.word_embedding = self.llm.model.get_input_embeddings() - - - def encode_inputs(self, samples: Batch): - batch_size = len(samples['question']) - questions = self.tokenizer(samples["question"], - add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], - add_special_tokens=False) - - # encode special tokens - eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding( - self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to( - self.llm_device)) - pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to( - self.llm_device)).unsqueeze(0) - return batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds - - def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs(samples) - batch_inputs_embeds = [] - batch_attention_mask = [] - for i in range(batch_size): - # Add bos & eos token - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat( - [bos_embeds, inputs_embeds], - dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) - - with torch.cuda.amp.autocast(dtype=torch.bfloat16): - outputs = self.llm.generate( - inputs_embeds=inputs_embeds, - max_new_tokens=max_new_tokens, - attention_mask=attention_mask, - # do_sample=True, - use_cache=True # IMPORTANT! - ) - pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - - return { - 'pred': pred, - 'label': samples['label'], - 'question': samples['question'], - 'desc': samples['desc'], - } - - -class GAT_LLAMA(nn.Module): - def __init__(self, hidden_channels: int, num_gnn_layers: int): - super().__init__() - - - self.llama2 = LLAMA2() - - print("Training LLAMA with LORA!") - self.llm = self.llama2.llm - self.llm_device = self.llama2.llm_device - self.llm = prepare_model_for_kbit_training(self.llm) - self.tokenizer = self.llama2.tokenizer - lora_r: int = 8 - lora_alpha: int = 16 - lora_dropout: float = 0.05 - lora_target_modules = [ - "q_proj", - "v_proj", - ] - config = LoraConfig( - r=lora_r, - lora_alpha=lora_alpha, - target_modules=lora_target_modules, - lora_dropout=lora_dropout, - bias="none", - task_type="CAUSAL_LM", - ) - self.llm = get_peft_model(self.llm, config) - - print('Finish loading LLAMA!') - - self.graph_encoder = torch_geometric.nn.models.GAT( - in_channels=1024, - out_channels=1024, - hidden_channels=hidden_channels, - num_layers=num_gnn_layers, - heads=4, - ).to(self.llm_device) - self.projector = nn.Sequential( - nn.Linear(1024, 2048), - nn.Sigmoid(), - nn.Linear(2048, 4096), - ).to(self.llm_device) - - self.word_embedding = self.llama2.word_embedding - - def encode_graphs(self, samples: Batch): - x = samples.x.to(self.llm_device) - edge_index = samples.edge_index.long().to(self.llm_device) - edge_attr = samples.edge_attr.to(self.llm_device) - n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - batch = samples.batch.to(self.llm_device) - # mean pooling - g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') - return g_embeds - - def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) - # encode labels - labels = self.tokenizer(samples.label, add_special_tokens=False) - # encode training specific special token - eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - - # encode graphs - graph_embeds = self.encode_graphs(samples) - graph_embeds = self.projector(graph_embeds) - batch_inputs_embeds = [] - batch_attention_mask = [] - batch_label_input_ids = [] - num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] - for i in range(batch_size): - # Add bos & eos token - label_input_ids = labels.input_ids[ - i][:max_new_tokens] + eos_tokens.input_ids - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids + label_input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - to_cat = [bos_embeds] - if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - label_input_ids = [IGNORE_INDEX - ] * (inputs_embeds.shape[0] - - len(label_input_ids)) + label_input_ids - batch_label_input_ids.append(label_input_ids) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - batch_label_input_ids[ - i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) - label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) - - with torch.cuda.amp.autocast(dtype=torch.bfloat16): - outputs = self.llm( - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - return_dict=True, - labels=label_input_ids, - ) - return outputs.loss - - def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llama2.encode_inputs(samples) - # encode graphs - graph_embeds = self.encode_graphs(samples) - graph_embeds = self.projector(graph_embeds) - - batch_inputs_embeds = [] - batch_attention_mask = [] - for i in range(batch_size): - # Add bos & eos token - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) - - with torch.cuda.amp.autocast(dtype=torch.bfloat16): - outputs = self.llm.generate( - inputs_embeds=inputs_embeds, - max_new_tokens=max_new_tokens, - attention_mask=attention_mask, - # do_sample=True, - use_cache=True # IMPORTANT! - ) - pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - - return { - 'pred': pred, - 'label': samples['label'], - 'question': samples['question'], - 'desc': samples['desc'], - } - - def print_trainable_params(self): - trainable_params = 0 - all_param = 0 - for _, param in self.named_parameters(): - num_params = param.numel() - - all_param += num_params - if param.requires_grad: - trainable_params += num_params - - return trainable_params, all_param - - def main(since: float, num_epochs: int, hidden_channels: int, num_gnn_layers: int, batch_size: int, lr: float): def adjust_learning_rate(param_group, LR, epoch): @@ -419,7 +124,7 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GAT_LLAMA(hidden_channels, num_gnn_layers) + model = GNN_LLM(hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -505,7 +210,7 @@ def minimal_demo(model, dataset): loader = DataLoader(test_dataset, batch_size=1, drop_last=False, pin_memory=True, shuffle=False) # define the pure pretrained LLM - pure_llm = LLAMA2() + pure_llm = LLM() # Step loop through the loader and run both models gnn_llm_hallucin_sum = 0 diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 7609ea0c6774..1b4d2623b8e5 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -146,7 +146,7 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): """This GNN+LLM implementation is based on the design from G-retriever. See `examples/llm_plus_gnn/g_retriever.py` - for an example. Original Paper: https://arxiv.org/abs/2402.07630 + for example usage. Original Paper: https://arxiv.org/abs/2402.07630 Args: llm_to_use (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. From 122e16f74b2fff8f71c92408f9c48f49a85da4df Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:07:54 -0700 Subject: [PATCH 213/752] done drafting, debugging time --- torch_geometric/nn/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index 9b70b8b73875..7a81aed797a1 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -28,7 +28,7 @@ from .pmlp import PMLP from .neural_fingerprint import NeuralFingerprint from .visnet import ViSNet - +from .gnn_llm import GNN_LLM # Deprecated: from torch_geometric.explain.algorithm.captum import (to_captum_input, captum_output_to_dicts) From b66df4b042b6a100cb962c17d6949aa098915e77 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:18:49 -0700 Subject: [PATCH 214/752] done drafting, debugging time --- torch_geometric/nn/models/gnn_llm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 1b4d2623b8e5..e7fe5693016f 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -7,7 +7,8 @@ from torch_geometric import seed_everything from torch_geometric.data import Batch from torch_geometric.utils import scatter -from torch_geometric.nn.models import GAT, BasicGNN +from torch_geometric.nn.models import GAT +from torch_geometric.nn.models.basic_gnn import BasicGNN BOS = '[INST]' @@ -164,7 +165,7 @@ class GNN_LLM(nn.Module): automatically allocate the number of gpus needed, given the available GPU memory of your GPUs (default :obj:`7`) gnn_to_use (BasicGNN): Please pass a valid model that extends - torch_geometric.nn.models.BasicGNN. (default: :obj:`GAT`) + torch_geometric.nn.models.basic_gnn.BasicGNN. (default: :obj:`GAT`) gnn_in_channels (int): (default: 1024) gnn_hidden_channels (int): (default: 1024) gnn_out_channels (int): (default: 1024) From 4e8ae7733d53cef1033ba59e164eb7f95181f4ed Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:31:44 -0700 Subject: [PATCH 215/752] debugging time... --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index fb246a34214e..43015a540565 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -124,7 +124,7 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GNN_LLM(hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) + model = GNN_LLM(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] From 23906ca8807a78993dcced0da5f176dba7a417d3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:38:35 -0700 Subject: [PATCH 216/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index e7fe5693016f..15be540ee0f2 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -49,7 +49,7 @@ def get_llm_kwargs(mem_needed): return kwargs class LLM(nn.Module): - def __init__(self, llm_name: str, num_params: int = 7, llm_dtype=torch.bfloat16): + def __init__(self, llm_name: str, llm_dtype=torch.bfloat16, num_params: int = 7): super().__init__() if llm_name == "llama2": self.printable_llm_name = "LLAMA2" From a062468e12719f4beacbe8ecbb0ff4efb76b673e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:42:53 -0700 Subject: [PATCH 217/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 15be540ee0f2..c3d51110d83c 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -186,7 +186,7 @@ def __init__(self, llm_to_use='llama2', use_lora_for_llm: bool = True, else: self.llm_to_use = LLM(llm_to_use, llm_dtype) self.llm = self.llm_to_use.llm - self.llm_dtype = self.llm_dtype + self.llm_dtype = llm_dtype if use_lora: print("Training our LLM with LORA!") self.llm = prepare_model_for_kbit_training(self.llm) From c84d957328b52a85f55ac8f4dd4fa89f8ae3bb7a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:43:45 -0700 Subject: [PATCH 218/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index c3d51110d83c..a5b71742df61 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -156,7 +156,7 @@ class GNN_LLM(nn.Module): please file an issue on https://github.com/pyg-team/pytorch_geometric and assign to puririshi98. (default: :obj:'llama2') - use_lora (bool): use LORA from peft for training the LLM. see + llm_use_lora (bool): use LORA from peft for training the LLM. see https://huggingface.co/docs/peft/en/index for details. llm_dtype (torch.dtype): The lower precision dtype to use for the LLM. (default :obj: `torch.bloat16`) @@ -173,7 +173,7 @@ class GNN_LLM(nn.Module): num_gnn_heads (int): Number of heads to use for BasicGNNs with the `heads` kwarg. (default: 4) """ - def __init__(self, llm_to_use='llama2', use_lora_for_llm: bool = True, + def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, llm_dtype=torch.bfloat16, num_llm_params: int = 7, gnn_to_use=GAT, gnn_in_channels: int = 1024, gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, @@ -187,7 +187,7 @@ def __init__(self, llm_to_use='llama2', use_lora_for_llm: bool = True, self.llm_to_use = LLM(llm_to_use, llm_dtype) self.llm = self.llm_to_use.llm self.llm_dtype = llm_dtype - if use_lora: + if llm_use_lora: print("Training our LLM with LORA!") self.llm = prepare_model_for_kbit_training(self.llm) lora_r: int = 8 From 73c735b78ab4ee21ab2a3f664571828b11c33d38 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 13:44:23 -0700 Subject: [PATCH 219/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a5b71742df61..da0f836b51c4 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -211,9 +211,9 @@ def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, print('Finished loading LLAMA!') self.graph_encoder = gnn_to_use( - in_channels=in_channels, - out_channels=out_channels, - hidden_channels=hidden_channels, + in_channels=gnn_in_channels, + out_channels=gnn_out_channels, + hidden_channels=gnn_hidden_channels, num_layers=num_gnn_layers, heads=num_gnn_heads, ).to(self.llm_device) From 272f3b3a5b50a8ce9b6cdb94235d1cad4a051800 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 14:06:29 -0700 Subject: [PATCH 220/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index da0f836b51c4..3d0bda0ac685 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -172,12 +172,15 @@ class GNN_LLM(nn.Module): num_gnn_layers (int): (default: 4) num_gnn_heads (int): Number of heads to use for BasicGNNs with the `heads` kwarg. (default: 4) + mlp_hidden_dim (int): (default: 2048) + mlp_hidden_dim (int): (default: 4096) """ def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, llm_dtype=torch.bfloat16, num_llm_params: int = 7, gnn_to_use=GAT, gnn_in_channels: int = 1024, gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, - num_gnn_layers: int = 4, num_gnn_heads: int = 4,): + num_gnn_layers: int = 4, num_gnn_heads: int = 4, + mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096,): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) @@ -217,10 +220,12 @@ def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, num_layers=num_gnn_layers, heads=num_gnn_heads, ).to(self.llm_device) + # For the MLP Projection + mlp_hidden_dim = gnn_out_channels self.projector = nn.Sequential( - nn.Linear(1024, 2048), + nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), - nn.Linear(2048, 4096), + nn.Linear(mlp_hidden_dim, mlp_out_dim), ).to(self.llm_device) self.word_embedding = self.llm_to_use.word_embedding From 05f2c6fd5ba2f21e3c9f8273edf0c3576d97411e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 14:27:27 -0700 Subject: [PATCH 221/752] debugging time... --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 3d0bda0ac685..f4843a4b4db6 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -49,7 +49,7 @@ def get_llm_kwargs(mem_needed): return kwargs class LLM(nn.Module): - def __init__(self, llm_name: str, llm_dtype=torch.bfloat16, num_params: int = 7): + def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, num_params: int = 7): super().__init__() if llm_name == "llama2": self.printable_llm_name = "LLAMA2" From 2bc97a133efef18cf120bf7e5ab22735a945def9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 15:08:17 -0700 Subject: [PATCH 222/752] llamas working, will try gemma once im done running llama --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index f4843a4b4db6..fa5ca1f18ced 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -175,7 +175,7 @@ class GNN_LLM(nn.Module): mlp_hidden_dim (int): (default: 2048) mlp_hidden_dim (int): (default: 4096) """ - def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, + def __init__(self, llm_to_use='gemma', llm_use_lora: bool = True, llm_dtype=torch.bfloat16, num_llm_params: int = 7, gnn_to_use=GAT, gnn_in_channels: int = 1024, gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, From 47b99f296cacc2ea706a1154a9af24d816ef4c16 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 15:14:26 -0700 Subject: [PATCH 223/752] llamas working, will try gemma once im done running llama --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index fa5ca1f18ced..4075f47232f0 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -173,7 +173,7 @@ class GNN_LLM(nn.Module): num_gnn_heads (int): Number of heads to use for BasicGNNs with the `heads` kwarg. (default: 4) mlp_hidden_dim (int): (default: 2048) - mlp_hidden_dim (int): (default: 4096) + mlp_out_dim (int): (default: 4096) """ def __init__(self, llm_to_use='gemma', llm_use_lora: bool = True, llm_dtype=torch.bfloat16, num_llm_params: int = 7, From 45e6db919a63fca01feb444ad1160850e36e124b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 5 Apr 2024 15:46:48 -0700 Subject: [PATCH 224/752] testing gemma --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- torch_geometric/nn/models/gnn_llm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 43015a540565..e286fe1386fd 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -199,7 +199,7 @@ def adjust_learning_rate(param_group, LR, epoch): acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') # save model - torch.save(model, "gat_llama.pt") + torch.save(model, "gnn_llm.pt") return prep_time, dataset, model def minimal_demo(model, dataset): @@ -274,7 +274,7 @@ def minimal_demo(model, dataset): print("E2E time (e2e_time) =", e2e_time) print("E2E time minus Prep Time =", e2e_time - prep_time) else: - model = torch.load("gat_llama.pt") + model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() print("Would you like a minimal demo showcasing how GNN+LLM can solve LLM hallucinations?") user_input = str(input("(y/n):")).lower() diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 4075f47232f0..f9cf0b8d2ad4 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -180,7 +180,7 @@ def __init__(self, llm_to_use='gemma', llm_use_lora: bool = True, gnn_to_use=GAT, gnn_in_channels: int = 1024, gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, num_gnn_layers: int = 4, num_gnn_heads: int = 4, - mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096,): + mlp_hidden_dim: int = 2048, mlp_out_dim: int = 3072,): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) From 4e301d41732e8aa722e5440b17096660f4202238 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 09:28:41 -0700 Subject: [PATCH 225/752] gemma performed, worse, may need tuning, will leave for community sprint --- torch_geometric/nn/models/gnn_llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index f9cf0b8d2ad4..909ab9f683af 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -175,12 +175,12 @@ class GNN_LLM(nn.Module): mlp_hidden_dim (int): (default: 2048) mlp_out_dim (int): (default: 4096) """ - def __init__(self, llm_to_use='gemma', llm_use_lora: bool = True, + def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, llm_dtype=torch.bfloat16, num_llm_params: int = 7, gnn_to_use=GAT, gnn_in_channels: int = 1024, gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, num_gnn_layers: int = 4, num_gnn_heads: int = 4, - mlp_hidden_dim: int = 2048, mlp_out_dim: int = 3072,): + mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096,): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) From 7cb92505763d1ebf58f049768efc4d4c7b4cbd66 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:45:35 +0000 Subject: [PATCH 226/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 33 +++++++++----- torch_geometric/nn/models/__init__.py | 48 ++++----------------- torch_geometric/nn/models/gnn_llm.py | 62 +++++++++++++++------------ 3 files changed, 65 insertions(+), 78 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index e286fe1386fd..a88b8e2905ff 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -12,6 +12,7 @@ import re import time from os import path + import pandas as pd import torch import torch.nn as nn @@ -24,9 +25,8 @@ from torch_geometric import seed_everything from torch_geometric.data import Batch, DataLoader from torch_geometric.datasets import WebQSPDataset +from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM from torch_geometric.utils import scatter -from torch_geometric.nn.models.gnn_llm import LLM, GNN_LLM - BOS = '[INST]' EOS_USER = '[/INST]' @@ -38,13 +38,14 @@ pad_token_id = 0 padding_side = 'left' + def detect_hallucinate(pred, label): try: pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(pred[0], label)) > 0 hallucination = not correct_hit return hallucination - except: # noqa + except: # noqa return "skip" @@ -91,6 +92,7 @@ def compute_accuracy(eval_output): return hit + def main(since: float, num_epochs: int, hidden_channels: int, num_gnn_layers: int, batch_size: int, lr: float): def adjust_learning_rate(param_group, LR, epoch): @@ -124,7 +126,8 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GNN_LLM(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) + model = GNN_LLM(gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -202,13 +205,14 @@ def adjust_learning_rate(param_group, LR, epoch): torch.save(model, "gnn_llm.pt") return prep_time, dataset, model + def minimal_demo(model, dataset): # Step 1: Define a single batch size test loader idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] # batch size 1 loader for simplicity loader = DataLoader(test_dataset, batch_size=1, drop_last=False, - pin_memory=True, shuffle=False) + pin_memory=True, shuffle=False) # define the pure pretrained LLM pure_llm = LLM() @@ -225,7 +229,8 @@ def minimal_demo(model, dataset): gnn_llm_pred = gnn_llm_out['pred'][0] pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) + pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, + correct_answer) if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # skipping since hard to evaluate if the answer's are hallucinations continue @@ -237,15 +242,17 @@ def minimal_demo(model, dataset): final_prnt_str += "Correct Answer: " + correct_answer + "\n" final_prnt_str += "Pure LLM Prediction: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Prediction:" + gnn_llm_pred + "\n" - final_prnt_str += "#"*20 + "\n" + final_prnt_str += "#" * 20 + "\n" print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) - percent = 100.0 * round(1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum) , 2) + percent = 100.0 * round(1 - + (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) print(f"GNN reduces hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") print("Instances where GNN solves the hallucinations of Pure LLMs:") print(final_prnt_str) + if __name__ == "__main__": # check if saved model if path.exists("gat_llama.pt"): @@ -265,8 +272,10 @@ def minimal_demo(model, dataset): parser.add_argument('--batch_size', type=int, default=4) args = parser.parse_args() since = time.time() - prep_time, dataset, model = main(since, args.epochs, args.hidden_channels, - args.num_gnn_layers, args.batch_size, args.lr) + prep_time, dataset, model = main(since, args.epochs, + args.hidden_channels, + args.num_gnn_layers, args.batch_size, + args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -276,7 +285,9 @@ def minimal_demo(model, dataset): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - print("Would you like a minimal demo showcasing how GNN+LLM can solve LLM hallucinations?") + print( + "Would you like a minimal demo showcasing how GNN+LLM can solve LLM hallucinations?" + ) user_input = str(input("(y/n):")).lower() if user_input == "y": minimal_demo(model, dataset) diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index 7a81aed797a1..a89c9c9a9539 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -34,45 +34,13 @@ captum_output_to_dicts) __all__ = classes = [ - 'MLP', - 'GCN', - 'GraphSAGE', - 'GIN', - 'GAT', - 'PNA', - 'EdgeCNN', - 'JumpingKnowledge', - 'MetaLayer', - 'Node2Vec', - 'DeepGraphInfomax', - 'InnerProductDecoder', - 'GAE', - 'VGAE', - 'ARGA', - 'ARGVA', - 'SignedGCN', - 'RENet', - 'GraphUNet', - 'SchNet', - 'DimeNet', - 'DimeNetPlusPlus', - 'to_captum_model', - 'to_captum_input', - 'captum_output_to_dicts', - 'MetaPath2Vec', - 'DeepGCNLayer', - 'TGNMemory', - 'LabelPropagation', - 'CorrectAndSmooth', - 'AttentiveFP', - 'RECT_L', - 'LINKX', - 'LightGCN', - 'MaskLabel', - 'GroupAddRev', - 'GNNFF', - 'PMLP', - 'NeuralFingerprint', - 'ViSNet', + 'MLP', 'GCN', 'GraphSAGE', 'GIN', 'GAT', 'PNA', 'EdgeCNN', + 'JumpingKnowledge', 'MetaLayer', 'Node2Vec', 'DeepGraphInfomax', + 'InnerProductDecoder', 'GAE', 'VGAE', 'ARGA', 'ARGVA', 'SignedGCN', + 'RENet', 'GraphUNet', 'SchNet', 'DimeNet', 'DimeNetPlusPlus', + 'to_captum_model', 'to_captum_input', 'captum_output_to_dicts', + 'MetaPath2Vec', 'DeepGCNLayer', 'TGNMemory', 'LabelPropagation', + 'CorrectAndSmooth', 'AttentiveFP', 'RECT_L', 'LINKX', 'LightGCN', + 'MaskLabel', 'GroupAddRev', 'GNNFF', 'PMLP', 'NeuralFingerprint', 'ViSNet', 'GNN_LLM' ] diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 909ab9f683af..09b5cabf8de2 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -6,10 +6,9 @@ import torch_geometric from torch_geometric import seed_everything from torch_geometric.data import Batch -from torch_geometric.utils import scatter from torch_geometric.nn.models import GAT from torch_geometric.nn.models.basic_gnn import BasicGNN - +from torch_geometric.utils import scatter BOS = '[INST]' EOS_USER = '[/INST]' @@ -22,6 +21,7 @@ pad_token_id = 0 padding_side = 'left' + def get_llm_kwargs(mem_needed): assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" avail_gpus = torch.cuda.device_count() @@ -48,8 +48,10 @@ def get_llm_kwargs(mem_needed): kwargs["device_map"] = "auto" return kwargs + class LLM(nn.Module): - def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, num_params: int = 7): + def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, + num_params: int = 7): super().__init__() if llm_name == "llama2": self.printable_llm_name = "LLAMA2" @@ -70,12 +72,11 @@ def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, num_param self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( - self.huggingface_str, torch_dtype=self.llm_dtype, low_cpu_mem_usage=True, - **kwargs) + self.huggingface_str, torch_dtype=self.llm_dtype, + low_cpu_mem_usage=True, **kwargs) self.llm_device = self.llm.device self.word_embedding = self.llm.model.get_input_embeddings() - def encode_inputs(self, samples: Batch): batch_size = len(samples['question']) questions = self.tokenizer(samples["question"], @@ -96,7 +97,8 @@ def encode_inputs(self, samples: Batch): def inference(self, samples: Batch): # this function is for comparing a pretrained LLM to a trained GNN_LLM - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs(samples) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs( + samples) batch_inputs_embeds = [] batch_attention_mask = [] for i in range(batch_size): @@ -106,9 +108,7 @@ def inference(self, samples: Batch): i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat( - [bos_embeds, inputs_embeds], - dim=0) + inputs_embeds = torch.cat([bos_embeds, inputs_embeds], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -123,8 +123,7 @@ def inference(self, samples: Batch): inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm.generate( @@ -145,7 +144,7 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): - """This GNN+LLM implementation is based on the design from + """This GNN+LLM implementation is based on the design from G-retriever. See `examples/llm_plus_gnn/g_retriever.py` for example usage. Original Paper: https://arxiv.org/abs/2402.07630 Args: @@ -156,7 +155,7 @@ class GNN_LLM(nn.Module): please file an issue on https://github.com/pyg-team/pytorch_geometric and assign to puririshi98. (default: :obj:'llama2') - llm_use_lora (bool): use LORA from peft for training the LLM. see + llm_use_lora (bool): use LORA from peft for training the LLM. see https://huggingface.co/docs/peft/en/index for details. llm_dtype (torch.dtype): The lower precision dtype to use for the LLM. (default :obj: `torch.bloat16`) @@ -175,12 +174,21 @@ class GNN_LLM(nn.Module): mlp_hidden_dim (int): (default: 2048) mlp_out_dim (int): (default: 4096) """ - def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, - llm_dtype=torch.bfloat16, num_llm_params: int = 7, - gnn_to_use=GAT, gnn_in_channels: int = 1024, - gnn_hidden_channels: int = 1024, gnn_out_channels: int = 1024, - num_gnn_layers: int = 4, num_gnn_heads: int = 4, - mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096,): + def __init__( + self, + llm_to_use='llama2', + llm_use_lora: bool = True, + llm_dtype=torch.bfloat16, + num_llm_params: int = 7, + gnn_to_use=GAT, + gnn_in_channels: int = 1024, + gnn_hidden_channels: int = 1024, + gnn_out_channels: int = 1024, + num_gnn_layers: int = 4, + num_gnn_heads: int = 4, + mlp_hidden_dim: int = 2048, + mlp_out_dim: int = 4096, + ): super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) @@ -221,7 +229,7 @@ def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, heads=num_gnn_heads, ).to(self.llm_device) # For the MLP Projection - mlp_hidden_dim = gnn_out_channels + mlp_hidden_dim = gnn_out_channels self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), @@ -241,7 +249,8 @@ def encode_graphs(self, samples: Batch): return g_embeds def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs( + samples) # encode labels labels = self.tokenizer(samples.label, add_special_tokens=False) # encode training specific special token @@ -288,8 +297,7 @@ def forward(self, samples: Batch): inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) @@ -303,7 +311,8 @@ def forward(self, samples: Batch): return outputs.loss def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) + batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs( + samples) # encode graphs graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) @@ -334,8 +343,7 @@ def inference(self, samples: Batch): inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm.generate( From 3889838a85b6370c284e2d4bab053939271fde83 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 10:47:15 -0700 Subject: [PATCH 227/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 909ab9f683af..d08c2896e1e0 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -1,6 +1,10 @@ import torch import torch.nn as nn -from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training +try: + from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training + WITH_PEFT = True +except: + WITH_PEFT = False from transformers import AutoModelForCausalLM, AutoTokenizer import torch_geometric @@ -191,6 +195,8 @@ def __init__(self, llm_to_use='llama2', llm_use_lora: bool = True, self.llm = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: + if not WITH_PEFT: + raise ImportError("To use LORA, please `pip install peft`.") print("Training our LLM with LORA!") self.llm = prepare_model_for_kbit_training(self.llm) lora_r: int = 8 From 209eaa60c65d03b041c9c5f9ae8bcab5e2aa7fe9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:48:39 +0000 Subject: [PATCH 228/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ab19293a95e6..9794aaf80e17 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -1,7 +1,12 @@ import torch import torch.nn as nn + try: - from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training + from peft import ( + LoraConfig, + get_peft_model, + prepare_model_for_kbit_training, + ) WITH_PEFT = True except: WITH_PEFT = False From e1568cdf48d0c5ba3af99cd22335c5d37f833493 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 10:59:23 -0700 Subject: [PATCH 229/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ab19293a95e6..16e3bab8db90 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -5,7 +5,12 @@ WITH_PEFT = True except: WITH_PEFT = False -from transformers import AutoModelForCausalLM, AutoTokenizer + +try: + from transformers import AutoModelForCausalLM, AutoTokenizer + WITH_TRANSFORMERS = True +except ImportError as e: # noqa + WITH_TRANSFORMERS = False import torch_geometric from torch_geometric import seed_everything @@ -194,6 +199,8 @@ def __init__( mlp_out_dim: int = 4096, ): super().__init__() + if not WITH_TRANSFORMERS: + raise ImportError("To use GNN_LLM, please `pip install transformers`.") if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) elif 'gemma' in llm_to_use.lower(): From dd227b1c17a54d7a50625a946e273436ef8745f9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:00:53 +0000 Subject: [PATCH 230/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 2df68ba38a4e..a26d4391862e 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -205,7 +205,8 @@ def __init__( ): super().__init__() if not WITH_TRANSFORMERS: - raise ImportError("To use GNN_LLM, please `pip install transformers`.") + raise ImportError( + "To use GNN_LLM, please `pip install transformers`.") if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) elif 'gemma' in llm_to_use.lower(): From b8403055f84543c98bbb8e121b2f3311309a80ff Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 11:12:00 -0700 Subject: [PATCH 231/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index a88b8e2905ff..5e01dacbee47 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -16,17 +16,14 @@ import pandas as pd import torch import torch.nn as nn -from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm -from transformers import AutoModelForCausalLM, AutoTokenizer import torch_geometric from torch_geometric import seed_everything -from torch_geometric.data import Batch, DataLoader +from torch_geometric.data import DataLoader from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM -from torch_geometric.utils import scatter BOS = '[INST]' EOS_USER = '[/INST]' @@ -232,7 +229,7 @@ def minimal_demo(model, dataset): pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": - # skipping since hard to evaluate if the answer's are hallucinations + # skipping since hard to evaluate if the answer is a hallucination continue gnn_llm_hallucin_sum += bool(gnn_llm_hallucinates) pure_llm_hallucin_sum += bool(pure_llm_hallucinates) From ea01499cf5007d0aac5850321b0957c783a40fbe Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 11:28:28 -0700 Subject: [PATCH 232/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 -- torch_geometric/nn/models/gnn_llm.py | 9 ++++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5e01dacbee47..26ba41cf6e35 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -15,11 +15,9 @@ import pandas as pd import torch -import torch.nn as nn from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm -import torch_geometric from torch_geometric import seed_everything from torch_geometric.data import DataLoader from torch_geometric.datasets import WebQSPDataset diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a26d4391862e..ce208ea6dd35 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -18,10 +18,8 @@ WITH_TRANSFORMERS = False import torch_geometric -from torch_geometric import seed_everything from torch_geometric.data import Batch from torch_geometric.nn.models import GAT -from torch_geometric.nn.models.basic_gnn import BasicGNN from torch_geometric.utils import scatter BOS = '[INST]' @@ -107,12 +105,13 @@ def encode_inputs(self, samples: Batch): pad_embeds = self.word_embedding( torch.tensor(self.tokenizer.pad_token_id).to( self.llm_device)).unsqueeze(0) - return batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds + return (batch_size, questions, descriptions, eos_user_tokens, + bos_embeds, pad_embeds) def inference(self, samples: Batch): # this function is for comparing a pretrained LLM to a trained GNN_LLM - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.encode_inputs( - samples) + batch_size, questions, descriptions, eos_user_tokens, \ + bos_embeds, pad_embeds = self.encode_inputs(samples) batch_inputs_embeds = [] batch_attention_mask = [] for i in range(batch_size): From 5062e8af35d12be50f98f1e2be58e7d7273370e4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:29:33 +0000 Subject: [PATCH 233/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ce208ea6dd35..810a917ec4d3 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -106,7 +106,7 @@ def encode_inputs(self, samples: Batch): torch.tensor(self.tokenizer.pad_token_id).to( self.llm_device)).unsqueeze(0) return (batch_size, questions, descriptions, eos_user_tokens, - bos_embeds, pad_embeds) + bos_embeds, pad_embeds) def inference(self, samples: Batch): # this function is for comparing a pretrained LLM to a trained GNN_LLM From ffb294976895d0911a3980234eb8df1484500aef Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 11:33:08 -0700 Subject: [PATCH 234/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 3 ++- torch_geometric/nn/models/gnn_llm.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 26ba41cf6e35..1f8416d6838a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -281,7 +281,8 @@ def minimal_demo(model, dataset): model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() print( - "Would you like a minimal demo showcasing how GNN+LLM can solve LLM hallucinations?" + "Would you like a minimal demo showcasing how \ + GNN+LLM can solve LLM hallucinations?" ) user_input = str(input("(y/n):")).lower() if user_input == "y": diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ce208ea6dd35..18a10ed50d7a 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -8,7 +8,7 @@ prepare_model_for_kbit_training, ) WITH_PEFT = True -except: +except ImportError as e: WITH_PEFT = False try: @@ -267,8 +267,8 @@ def encode_graphs(self, samples: Batch): return g_embeds def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs( - samples) + batch_size, questions, descriptions, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) # encode labels labels = self.tokenizer(samples.label, add_special_tokens=False) # encode training specific special token @@ -329,8 +329,8 @@ def forward(self, samples: Batch): return outputs.loss def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds = self.llm_to_use.encode_inputs( - samples) + batch_size, questions, descriptions, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) # encode graphs graph_embeds = self.encode_graphs(samples) graph_embeds = self.projector(graph_embeds) From 232e437d133c6c2a1ee4b85ac89b09bd6dfe845d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 11:33:23 -0700 Subject: [PATCH 235/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 18a10ed50d7a..7de33d844d00 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -17,7 +17,6 @@ except ImportError as e: # noqa WITH_TRANSFORMERS = False -import torch_geometric from torch_geometric.data import Batch from torch_geometric.nn.models import GAT from torch_geometric.utils import scatter From deecfec75b55fa8158f45574f0217e7de60a2854 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:34:35 +0000 Subject: [PATCH 236/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1f8416d6838a..feea242ea1ef 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -280,10 +280,8 @@ def minimal_demo(model, dataset): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - print( - "Would you like a minimal demo showcasing how \ - GNN+LLM can solve LLM hallucinations?" - ) + print("Would you like a minimal demo showcasing how \ + GNN+LLM can solve LLM hallucinations?") user_input = str(input("(y/n):")).lower() if user_input == "y": minimal_demo(model, dataset) From a058bd5b3be9d07515f4450365734acf8286da79 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 11:38:54 -0700 Subject: [PATCH 237/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a2e15260455c..ef9c2357e4e0 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -8,7 +8,7 @@ prepare_model_for_kbit_training, ) WITH_PEFT = True -except ImportError as e: +except ImportError as e: # noqa WITH_PEFT = False try: From 98332c20305cf07d5e1c1d560a3b2797d7ba2e03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:40:09 +0000 Subject: [PATCH 238/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ef9c2357e4e0..cd856b34d0b6 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -8,7 +8,7 @@ prepare_model_for_kbit_training, ) WITH_PEFT = True -except ImportError as e: # noqa +except ImportError as e: # noqa WITH_PEFT = False try: From d7c7f0ab931ca681bfdb9c9dbb36c1ecb59c2a80 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 12:24:14 -0700 Subject: [PATCH 239/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index ef9c2357e4e0..539869c1cac7 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -156,9 +156,9 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): - """This GNN+LLM implementation is based on the design from - G-retriever. See `examples/llm_plus_gnn/g_retriever.py` - for example usage. Original Paper: https://arxiv.org/abs/2402.07630 + r"""This GNN+LLM implementation is based on G-retriever. + See `examples/llm_plus_gnn/g_retriever.py` for example usage. + Original Paper: `_ Args: llm_to_use (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. From 133b6fecc5bb5564dd2f3bfefca474718e0147cb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 8 Apr 2024 12:47:59 -0700 Subject: [PATCH 240/752] cleaning up --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 88432d24eae8..f1d4012662fc 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -157,8 +157,8 @@ def inference(self, samples: Batch): class GNN_LLM(nn.Module): r"""This GNN+LLM implementation is based on G-retriever. + Original Paper: `_. See `examples/llm_plus_gnn/g_retriever.py` for example usage. - Original Paper: `_ Args: llm_to_use (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. From 963aa4a20196b24e94b8bcacd6f5b6a0ba31e026 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 19:49:04 +0000 Subject: [PATCH 241/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index f1d4012662fc..2ce043de3216 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -159,6 +159,7 @@ class GNN_LLM(nn.Module): r"""This GNN+LLM implementation is based on G-retriever. Original Paper: `_. See `examples/llm_plus_gnn/g_retriever.py` for example usage. + Args: llm_to_use (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. From f47552be609d41f5b35b53176d881d9116a62c5c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 9 Apr 2024 14:42:53 -0700 Subject: [PATCH 242/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index feea242ea1ef..26867681206b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -197,7 +197,9 @@ def adjust_learning_rate(param_group, LR, epoch): acc = compute_accuracy(eval_output) print(f'Test Acc {acc}') # save model + print("Saving Model...") torch.save(model, "gnn_llm.pt") + print("Done!") return prep_time, dataset, model From 86b301f182bba1e6f568e2f7491ec890bcb4b1ad Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 9 Apr 2024 15:08:20 -0700 Subject: [PATCH 243/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 26867681206b..4c1f1fc30fe6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -277,8 +277,8 @@ def minimal_demo(model, dataset): torch.cuda.reset_max_memory_allocated() gc.collect() e2e_time = round(time.time() - since, 2) - print("E2E time (e2e_time) =", e2e_time) - print("E2E time minus Prep Time =", e2e_time - prep_time) + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("E2E time minus Prep Time =", e2e_time - prep_time, "seconds") else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() From c3cf07609b54a9b075f425de9256b0a3bb35826c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 9 Apr 2024 16:43:20 -0700 Subject: [PATCH 244/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 4c1f1fc30fe6..64ddae85885f 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -235,7 +235,7 @@ def minimal_demo(model, dataset): pure_llm_hallucin_sum += bool(pure_llm_hallucinates) # showcase LLM hallucinations solved by GNN if pure_llm_hallucinates and not gnn_llm_hallucinates: - final_prnt_str += "Question: " + question + "\n" + final_prnt_str += "Prompt: " + question + "\n" final_prnt_str += "Correct Answer: " + correct_answer + "\n" final_prnt_str += "Pure LLM Prediction: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Prediction:" + gnn_llm_pred + "\n" From 0267f3b9b9403db4ed0cea7eff2853814526a0f1 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 9 Apr 2024 16:56:02 -0700 Subject: [PATCH 245/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 64ddae85885f..add153686ff2 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -38,6 +38,7 @@ def detect_hallucinate(pred, label): try: pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(pred[0], label)) > 0 + correct_hit = correct_hit or any([label_i in pred.lower() for label_i in label.split('|')]) hallucination = not correct_hit return hallucination except: # noqa @@ -246,7 +247,7 @@ def minimal_demo(model, dataset): (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) print(f"GNN reduces hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") - print("Instances where GNN solves the hallucinations of Pure LLMs:") + print("Potential instances where GNN solves the hallucinations of LLM:") print(final_prnt_str) From 8fae5cb82f970db20b0d4788ef25af3f09edb2e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 23:57:30 +0000 Subject: [PATCH 246/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index add153686ff2..70c584af6451 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -38,7 +38,8 @@ def detect_hallucinate(pred, label): try: pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(pred[0], label)) > 0 - correct_hit = correct_hit or any([label_i in pred.lower() for label_i in label.split('|')]) + correct_hit = correct_hit or any( + [label_i in pred.lower() for label_i in label.split('|')]) hallucination = not correct_hit return hallucination except: # noqa From e352692684bea424c91412d7bf939c75821f577f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 9 Apr 2024 17:43:22 -0700 Subject: [PATCH 247/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index add153686ff2..817796dfd583 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -284,7 +284,7 @@ def minimal_demo(model, dataset): model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() print("Would you like a minimal demo showcasing how \ - GNN+LLM can solve LLM hallucinations?") + GNN+LLM can solve LLM hallucinations?") user_input = str(input("(y/n):")).lower() if user_input == "y": minimal_demo(model, dataset) From 6bd0ce5babca8ac289d8291fba0f5ab751ae91c7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 08:30:02 -0700 Subject: [PATCH 248/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index a94c34cc61be..ee1eb643c5c0 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -36,8 +36,8 @@ def detect_hallucinate(pred, label): try: - pred = pred.split('[/s]')[0].strip().split('|') - correct_hit = len(re.findall(pred[0], label)) > 0 + split_pred = pred.split('[/s]')[0].strip().split('|') + correct_hit = len(re.findall(split_pred[0], label)) > 0 correct_hit = correct_hit or any( [label_i in pred.lower() for label_i in label.split('|')]) hallucination = not correct_hit From 18d56ded063b709d92040dd32d1671649074e041 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 10:04:40 -0700 Subject: [PATCH 249/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ee1eb643c5c0..6f88711e0db6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -254,7 +254,7 @@ def minimal_demo(model, dataset): if __name__ == "__main__": # check if saved model - if path.exists("gat_llama.pt"): + if path.exists("gnn_llm.pt"): print("Existing trained model found.") # ask if want to retrain or skip to demo print("Would you like to retrain?") From be3b729275833ca5fbfe9272257ed6d82a2b1f69 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 14:33:04 -0700 Subject: [PATCH 250/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 6f88711e0db6..f207ba730d94 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -239,8 +239,8 @@ def minimal_demo(model, dataset): if pure_llm_hallucinates and not gnn_llm_hallucinates: final_prnt_str += "Prompt: " + question + "\n" final_prnt_str += "Correct Answer: " + correct_answer + "\n" - final_prnt_str += "Pure LLM Prediction: " + pure_llm_pred + "\n" - final_prnt_str += "GNN+LLM Prediction:" + gnn_llm_pred + "\n" + final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" + final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) From 0ba6d7a2aa75b8bae111af06d5c9e7921f041811 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 14:33:19 -0700 Subject: [PATCH 251/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f207ba730d94..84d20e09b057 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -238,7 +238,7 @@ def minimal_demo(model, dataset): # showcase LLM hallucinations solved by GNN if pure_llm_hallucinates and not gnn_llm_hallucinates: final_prnt_str += "Prompt: " + question + "\n" - final_prnt_str += "Correct Answer: " + correct_answer + "\n" + final_prnt_str += "Label: " + correct_answer + "\n" final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" From 215174819ed0a8a931344b9f801021192258db45 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 14:36:29 -0700 Subject: [PATCH 252/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 84d20e09b057..b1ccaee53344 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -23,17 +23,6 @@ from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM -BOS = '[INST]' -EOS_USER = '[/INST]' -EOS = '[/s]' -IGNORE_INDEX = -100 -llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" -max_txt_len = 512 -max_new_tokens = 32 -pad_token_id = 0 -padding_side = 'left' - - def detect_hallucinate(pred, label): try: split_pred = pred.split('[/s]')[0].strip().split('|') From 95bf9126457a1a0ae8cf467637e40fd30136a1a0 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 14:36:39 -0700 Subject: [PATCH 253/752] cleaning up --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index b1ccaee53344..d1043eb51aab 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -23,6 +23,7 @@ from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM + def detect_hallucinate(pred, label): try: split_pred = pred.split('[/s]')[0].strip().split('|') From 27fcc13e5f5f84c5b18c702e06fadf2770a8fbcf Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 15:14:57 -0700 Subject: [PATCH 254/752] seeing how demo changes if i change max_new_tokens to 256 for llm --- torch_geometric/nn/models/gnn_llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 2ce043de3216..49bda58d8bd4 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -107,7 +107,7 @@ def encode_inputs(self, samples: Batch): return (batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds) - def inference(self, samples: Batch): + def inference(self, samples: Batch, max_out_tokens=256): # this function is for comparing a pretrained LLM to a trained GNN_LLM batch_size, questions, descriptions, eos_user_tokens, \ bos_embeds, pad_embeds = self.encode_inputs(samples) @@ -140,7 +140,7 @@ def inference(self, samples: Batch): with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm.generate( inputs_embeds=inputs_embeds, - max_new_tokens=max_new_tokens, + max_new_tokens=max_out_tokens, attention_mask=attention_mask, # do_sample=True, use_cache=True # IMPORTANT! From 8c20ed8cf3a813a11f6b7ae3ce2c212b1b7e169c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 17:34:01 -0700 Subject: [PATCH 255/752] seeing how demo changes if i change max_new_tokens to 256 for llm --- examples/llm_plus_gnn/g_retriever.py | 95 +++++++++++++++++++++------- torch_geometric/nn/models/gnn_llm.py | 60 +++++++++++++++++- 2 files changed, 132 insertions(+), 23 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d1043eb51aab..9f551b9c31e1 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -79,9 +79,8 @@ def compute_accuracy(eval_output): return hit - -def main(since: float, num_epochs: int, hidden_channels: int, - num_gnn_layers: int, batch_size: int, lr: float): +def train(since, num_epochs, hidden_channels, + num_gnn_layers, batch_size, lr, model=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -113,8 +112,9 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - model = GNN_LLM(gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers) + if num_gnn_layers is not None: + model = GNN_LLM(gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -190,12 +190,15 @@ def adjust_learning_rate(param_group, LR, epoch): print(f'Test Acc {acc}') # save model print("Saving Model...") - torch.save(model, "gnn_llm.pt") + if num_gnn_layers is not None: + torch.save(model, "gnn_llm.pt") + else: + torch.save(model, "llm.pt") print("Done!") return prep_time, dataset, model -def minimal_demo(model, dataset): +def minimal_demo(model, dataset, lr, epochs, batch_size): # Step 1: Define a single batch size test loader idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] @@ -210,6 +213,7 @@ def minimal_demo(model, dataset): pure_llm_hallucin_sum = 0 final_prnt_str = "" print("Checking LLM vs GNN+LLM for hallucinations...") + gnn_save_list = [] for batch in tqdm(loader): question = batch.question[0] correct_answer = batch.label[0] @@ -218,6 +222,7 @@ def minimal_demo(model, dataset): gnn_llm_pred = gnn_llm_out['pred'][0] pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) + gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": @@ -232,36 +237,83 @@ def minimal_demo(model, dataset): final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" - print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) percent = 100.0 * round(1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) print(f"GNN reduces hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") - print("Potential instances where GNN solves the hallucinations of LLM:") + print("Potential instances where GNN solves the hallucinations of LLM") + print(final_prnt_str) + print("Now we see how the LLM compares when finetuned?") + user_input = str(input("(y/n):")).lower() + since = time.time() + trained_hallucin_sum = 0 + final_prnt_str = "" + if path.exists("llm.pt"): + print("Existing finetuned LLAMA2 found.") + print("Would you like to retrain?") + user_input = str(input("(y/n):")).lower() + retrain = user_input == "y" + else: + retrain = True + if retrain: + print("Finetuning LLAMA2...") + _, dataset, pure_llm = train(since, epochs, None, + None, batch_size, lr, model=pure_llm) + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("Evaluating it...") + for batch in tqdm(enumerate(loader)): + question = batch.question[0] + correct_answer = batch.label[0] + gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] + if gnn_llm_hallucinates == "skip": + continue + pure_llm_pred = pure_llm.inference(batch)['pred'][0] + pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, + correct_answer) + if pure_llm_hallucinates == "skip": + continue + trained_llm_hallucin_sum += bool(pure_llm_hallucinates) + if pure_llm_hallucinates and not gnn_llm_hallucinates: + final_prnt_str += "Prompt: " + question + "\n" + final_prnt_str += "Label: " + correct_answer + "\n" + final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" + final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" + final_prnt_str += "#" * 20 + "\n" + print("After finetuning the LLM...") + print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) + print(f"GNN reduces hallucinations by: ~{percent}%") + print("Note: hallucinations detected by regex hence the ~") + print("Potential instances where GNN solves the hallucinations of LLM") print(final_prnt_str) + + + + + if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--hidden_channels', type=int, default=1024) + parser.add_argument('--num_gnn_layers', type=int, default=4) + parser.add_argument('--lr', type=float, default=1e-5) + parser.add_argument('--epochs', type=int, default=10) + parser.add_argument('--batch_size', type=int, default=4) + args = parser.parse_args() # check if saved model if path.exists("gnn_llm.pt"): print("Existing trained model found.") - # ask if want to retrain or skip to demo print("Would you like to retrain?") user_input = str(input("(y/n):")).lower() retrain = user_input == "y" else: retrain = True if retrain: - parser = argparse.ArgumentParser() - parser.add_argument('--hidden_channels', type=int, default=1024) - parser.add_argument('--num_gnn_layers', type=int, default=4) - parser.add_argument('--lr', type=float, default=1e-5) - parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=4) - args = parser.parse_args() since = time.time() - prep_time, dataset, model = main(since, args.epochs, + prep_time, dataset, model = train_LLM(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) @@ -274,8 +326,7 @@ def minimal_demo(model, dataset): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - print("Would you like a minimal demo showcasing how \ + print("Here is a minimal demo showcasing how \ GNN+LLM can solve LLM hallucinations?") - user_input = str(input("(y/n):")).lower() - if user_input == "y": - minimal_demo(model, dataset) + print("First comparing against a pretrained LLAMA2 model") + minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.yes) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 49bda58d8bd4..db71ea9f71a9 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -107,8 +107,66 @@ def encode_inputs(self, samples: Batch): return (batch_size, questions, descriptions, eos_user_tokens, bos_embeds, pad_embeds) + def forward(self, samples: Batch): + batch_size, questions, descriptions, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) + # encode labels + labels = self.tokenizer(samples.label, add_special_tokens=False) + # encode training specific special token + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[ + i][:max_new_tokens] + eos_tokens.input_ids + input_ids = descriptions.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=self.llm_dtype): + outputs = self.llm( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + return outputs.loss + def inference(self, samples: Batch, max_out_tokens=256): - # this function is for comparing a pretrained LLM to a trained GNN_LLM batch_size, questions, descriptions, eos_user_tokens, \ bos_embeds, pad_embeds = self.encode_inputs(samples) batch_inputs_embeds = [] From bf298dae380d2cf62c6471930f58b4e3938141ad Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 17:41:26 -0700 Subject: [PATCH 256/752] seeing how demo changes if i change max_new_tokens to 256 for llm --- examples/llm_plus_gnn/g_retriever.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9f551b9c31e1..276add331cba 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -80,7 +80,7 @@ def compute_accuracy(eval_output): return hit def train(since, num_epochs, hidden_channels, - num_gnn_layers, batch_size, lr, model=None): + num_gnn_layers, batch_size, lr, model=None, dataset=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -95,8 +95,8 @@ def adjust_learning_rate(param_group, LR, epoch): return lr seed_everything(42) - - dataset = WebQSPDataset() + if dataset is not None: + dataset = WebQSPDataset() idx_split = dataset.split_idxs # Step 1: Build Node Classification Dataset @@ -112,7 +112,7 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - if num_gnn_layers is not None: + if model is not None: model = GNN_LLM(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) @@ -259,9 +259,11 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") - _, dataset, pure_llm = train(since, epochs, None, - None, batch_size, lr, model=pure_llm) - print("E2E time (e2e_time) =", e2e_time, "seconds") + _, _, pure_llm = train(since, epochs, None, + None, batch_size, lr, model=pure_llm, dataset=dataset) + print("E2E time (e2e_time) =", e2e_time, "seconds") + else: + print("Evaluating it...") for batch in tqdm(enumerate(loader)): question = batch.question[0] From 8ea92c28473557f8918d79042c5fd1b8c5f71a7f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 17:42:40 -0700 Subject: [PATCH 257/752] seeing how demo changes if i change max_new_tokens to 256 for llm --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 276add331cba..048609b3b40b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -263,7 +263,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): None, batch_size, lr, model=pure_llm, dataset=dataset) print("E2E time (e2e_time) =", e2e_time, "seconds") else: - + pure_llm = torch.save("llm.pt") print("Evaluating it...") for batch in tqdm(enumerate(loader)): question = batch.question[0] From 06e9f68d6bd7fdab496deaec27fa6f0e8e548c6e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 00:43:44 +0000 Subject: [PATCH 258/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 29 +++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 048609b3b40b..e20b4c7f34bb 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -79,8 +79,9 @@ def compute_accuracy(eval_output): return hit -def train(since, num_epochs, hidden_channels, - num_gnn_layers, batch_size, lr, model=None, dataset=None): + +def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, lr, + model=None, dataset=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -114,7 +115,7 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is not None: model = GNN_LLM(gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers) + num_gnn_layers=num_gnn_layers) # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -259,8 +260,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") - _, _, pure_llm = train(since, epochs, None, - None, batch_size, lr, model=pure_llm, dataset=dataset) + _, _, pure_llm = train(since, epochs, None, None, batch_size, lr, + model=pure_llm, dataset=dataset) print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm = torch.save("llm.pt") @@ -273,8 +274,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): continue pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, - correct_answer) - if pure_llm_hallucinates == "skip": + correct_answer) + if pure_llm_hallucinates == "skip": continue trained_llm_hallucin_sum += bool(pure_llm_hallucinates) if pure_llm_hallucinates and not gnn_llm_hallucinates: @@ -292,11 +293,6 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print(final_prnt_str) - - - - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--hidden_channels', type=int, default=1024) @@ -316,9 +312,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if retrain: since = time.time() prep_time, dataset, model = train_LLM(since, args.epochs, - args.hidden_channels, - args.num_gnn_layers, args.batch_size, - args.lr) + args.hidden_channels, + args.num_gnn_layers, + args.batch_size, args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -331,4 +327,5 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Here is a minimal demo showcasing how \ GNN+LLM can solve LLM hallucinations?") print("First comparing against a pretrained LLAMA2 model") - minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.yes) + minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, + args.yes) From fe3e8dc01c0c16f18e909d0fd64165cd14c1dc1e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 17:43:46 -0700 Subject: [PATCH 259/752] seeing how demo changes if i change max_new_tokens to 256 for llm --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 048609b3b40b..d87b669dbe39 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -315,7 +315,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): retrain = True if retrain: since = time.time() - prep_time, dataset, model = train_LLM(since, args.epochs, + prep_time, dataset, model = trainM(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, args.lr) From 1095a4ebf59842d64750053fc6554d00c313dcdd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 00:45:53 +0000 Subject: [PATCH 260/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 622586fbd3f9..a2a084f06858 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -312,9 +312,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if retrain: since = time.time() prep_time, dataset, model = train(since, args.epochs, - args.hidden_channels, - args.num_gnn_layers, args.batch_size, - args.lr) + args.hidden_channels, + args.num_gnn_layers, args.batch_size, + args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 0cf11efb24a7a23fd2053554dff69bf975a4fbce Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 17:57:03 -0700 Subject: [PATCH 261/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 622586fbd3f9..1950f7d19e65 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -96,7 +96,7 @@ def adjust_learning_rate(param_group, LR, epoch): return lr seed_everything(42) - if dataset is not None: + if dataset is None: dataset = WebQSPDataset() idx_split = dataset.split_idxs @@ -113,7 +113,7 @@ def adjust_learning_rate(param_group, LR, epoch): drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model - if model is not None: + if model is None: model = GNN_LLM(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From c916a716072e36c7c3214084f5e1720a0cce1d5e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 18:00:29 -0700 Subject: [PATCH 262/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 4 +++- torch_geometric/nn/models/gnn_llm.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 629ee8736ae2..565866b9ebb6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -219,7 +219,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): question = batch.question[0] correct_answer = batch.label[0] gnn_llm_out = model.inference(batch) - pure_llm_out = pure_llm.inference(batch) + # GNN+LLM only using 32 tokens to answer, give untrained LLM more + pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) gnn_llm_pred = gnn_llm_out['pred'][0] pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) @@ -272,6 +273,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] if gnn_llm_hallucinates == "skip": continue + # 32 to match GNN pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index db71ea9f71a9..3f93b95d4a80 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -166,7 +166,7 @@ def forward(self, samples: Batch): ) return outputs.loss - def inference(self, samples: Batch, max_out_tokens=256): + def inference(self, samples: Batch, max_out_tokens=max_new_tokens): batch_size, questions, descriptions, eos_user_tokens, \ bos_embeds, pad_embeds = self.encode_inputs(samples) batch_inputs_embeds = [] From d7d95f4c38c9e94c15b9f5ab06764e5d3b937e30 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 10 Apr 2024 18:48:07 -0700 Subject: [PATCH 263/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 565866b9ebb6..09f8151bada9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -329,5 +329,4 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Here is a minimal demo showcasing how \ GNN+LLM can solve LLM hallucinations?") print("First comparing against a pretrained LLAMA2 model") - minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, - args.yes) + minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size) From e6b16627a8a536c5eb9f6dd332110bab7174c6fa Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 02:12:55 -0700 Subject: [PATCH 264/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 09f8151bada9..18973cd9a85b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -248,7 +248,6 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Potential instances where GNN solves the hallucinations of LLM") print(final_prnt_str) print("Now we see how the LLM compares when finetuned?") - user_input = str(input("(y/n):")).lower() since = time.time() trained_hallucin_sum = 0 final_prnt_str = "" From a1cddd1f6788256b1edf9fbf5034be16990eb398 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 07:44:31 -0700 Subject: [PATCH 265/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 18973cd9a85b..3a7943efc621 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -127,10 +127,11 @@ def adjust_learning_rate(param_group, LR, epoch): }, ], betas=(0.9, 0.95)) grad_steps = 2 - trainable_params, all_param = model.print_trainable_params() - print(f"trainable params: {trainable_params} || \ - all params: {all_param} || \ - trainable%: {100 * trainable_params / all_param}") + if model is None: + trainable_params, all_param = model.print_trainable_params() + print(f"trainable params: {trainable_params} || \ + all params: {all_param} || \ + trainable%: {100 * trainable_params / all_param}") # Step 4 Training for epoch in range(num_epochs): From 5afafc8a6d096d45f2a0552c84026ade5a75d0c1 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 08:21:00 -0700 Subject: [PATCH 266/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3a7943efc621..29926ee21d93 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -216,6 +216,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): final_prnt_str = "" print("Checking LLM vs GNN+LLM for hallucinations...") gnn_save_list = [] + untuned_llm_save_list = [] for batch in tqdm(loader): question = batch.question[0] correct_answer = batch.label[0] @@ -228,6 +229,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) + untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # skipping since hard to evaluate if the answer is a hallucination continue @@ -251,6 +253,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Now we see how the LLM compares when finetuned?") since = time.time() trained_hallucin_sum = 0 + untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" if path.exists("llm.pt"): print("Existing finetuned LLAMA2 found.") @@ -271,9 +274,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): question = batch.question[0] correct_answer = batch.label[0] gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] - if gnn_llm_hallucinates == "skip": + untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] + if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": continue - # 32 to match GNN pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) @@ -283,13 +286,18 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if pure_llm_hallucinates and not gnn_llm_hallucinates: final_prnt_str += "Prompt: " + question + "\n" final_prnt_str += "Label: " + correct_answer + "\n" - final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" + final_prnt_str += "Untuned LLM Output: " + untuned_llm_pred + "\n" + final_prnt_str += "Tuned LLM Output: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" print("After finetuning the LLM...") - print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + print("Total Tuned LLM Hallucinations:", untuned_llm_hallucin_sum) + print("Total Tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) - print(f"GNN reduces hallucinations by: ~{percent}%") + print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") + tuned_percent = 100.0 * round(1 - + (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) + print(f"GNN reduces tuned LLM hallucinations by: ~{tuned_percent}%") print("Note: hallucinations detected by regex hence the ~") print("Potential instances where GNN solves the hallucinations of LLM") print(final_prnt_str) From dc36114b47437ff9205038b3414f91e26bac69a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:22:05 +0000 Subject: [PATCH 267/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 29926ee21d93..04353ff03a0b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -295,8 +295,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Total Tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") - tuned_percent = 100.0 * round(1 - - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) + tuned_percent = 100.0 * round( + 1 - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) print(f"GNN reduces tuned LLM hallucinations by: ~{tuned_percent}%") print("Note: hallucinations detected by regex hence the ~") print("Potential instances where GNN solves the hallucinations of LLM") From 26d5f4a72cf81f13731441ccd26ff21981f17962 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 08:23:09 -0700 Subject: [PATCH 268/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 29926ee21d93..3297bb6a6852 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -250,7 +250,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("Note: hallucinations detected by regex hence the ~") print("Potential instances where GNN solves the hallucinations of LLM") print(final_prnt_str) - print("Now we see how the LLM compares when finetuned?") + print("Now we see how the LLM compares when finetuned...") since = time.time() trained_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum @@ -269,7 +269,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm = torch.save("llm.pt") - print("Evaluating it...") + print("Evaluating Tuned LLM...") for batch in tqdm(enumerate(loader)): question = batch.question[0] correct_answer = batch.label[0] From 21de3512a079897be5e0024cb614110661d50ad8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 08:23:39 -0700 Subject: [PATCH 269/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3297bb6a6852..d2f60e6be963 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -214,7 +214,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): gnn_llm_hallucin_sum = 0 pure_llm_hallucin_sum = 0 final_prnt_str = "" - print("Checking LLM vs GNN+LLM for hallucinations...") + print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") gnn_save_list = [] untuned_llm_save_list = [] for batch in tqdm(loader): From 96cba969b25840bdbc70b7fa9c0feea4f8fc62a3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 09:19:21 -0700 Subject: [PATCH 270/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index fbe882663f5e..a29995888ec5 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -233,8 +233,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # skipping since hard to evaluate if the answer is a hallucination continue - gnn_llm_hallucin_sum += bool(gnn_llm_hallucinates) - pure_llm_hallucin_sum += bool(pure_llm_hallucinates) + gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) + pure_llm_hallucin_sum += int(pure_llm_hallucinates) # showcase LLM hallucinations solved by GNN if pure_llm_hallucinates and not gnn_llm_hallucinates: final_prnt_str += "Prompt: " + question + "\n" @@ -282,7 +282,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): correct_answer) if pure_llm_hallucinates == "skip": continue - trained_llm_hallucin_sum += bool(pure_llm_hallucinates) + trained_llm_hallucin_sum += int(pure_llm_hallucinates) if pure_llm_hallucinates and not gnn_llm_hallucinates: final_prnt_str += "Prompt: " + question + "\n" final_prnt_str += "Label: " + correct_answer + "\n" From 5f153e9163c4217cf678ecc1416d03dc8c531a54 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 09:21:47 -0700 Subject: [PATCH 271/752] added finetuned llm baseline to demo --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 3f93b95d4a80..5749a1fac9a8 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -109,7 +109,7 @@ def encode_inputs(self, samples: Batch): def forward(self, samples: Batch): batch_size, questions, descriptions, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) + bos_embeds, pad_embeds = self.encode_inputs(samples) # encode labels labels = self.tokenizer(samples.label, add_special_tokens=False) # encode training specific special token From 4a78e72d8648a0502f27952d042a7fb9ef8f674d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 09:24:50 -0700 Subject: [PATCH 272/752] added finetuned llm baseline to demo --- torch_geometric/nn/models/gnn_llm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 5749a1fac9a8..9828ad50b05e 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -118,7 +118,6 @@ def forward(self, samples: Batch): batch_inputs_embeds = [] batch_attention_mask = [] batch_label_input_ids = [] - num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] for i in range(batch_size): # Add bos & eos token label_input_ids = labels.input_ids[ @@ -129,8 +128,6 @@ def forward(self, samples: Batch): inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] - if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) From e2fe0097eb54a78bdbaf8bb59f7acd0dbfa6aaf3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 09:42:00 -0700 Subject: [PATCH 273/752] added finetuned llm baseline to demo --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index a29995888ec5..b98034f82bf1 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -264,7 +264,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") - _, _, pure_llm = train(since, epochs, None, None, batch_size, lr, + _, _, pure_llm = train(since, 1, None, None, batch_size, lr, model=pure_llm, dataset=dataset) print("E2E time (e2e_time) =", e2e_time, "seconds") else: From cf9e2402f9d24ac637efc2b5dca1eeac95706698 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 11 Apr 2024 15:30:18 -0700 Subject: [PATCH 274/752] added finetuned llm baseline to demo --- torch_geometric/nn/models/gnn_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 9828ad50b05e..a102e2cb13e2 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -300,6 +300,7 @@ def __init__( hidden_channels=gnn_hidden_channels, num_layers=num_gnn_layers, heads=num_gnn_heads, + norm='batch_norm', ).to(self.llm_device) # For the MLP Projection mlp_hidden_dim = gnn_out_channels From c2ecf9378b7520a26179befb1510b1afd183950c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 09:47:28 -0700 Subject: [PATCH 275/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index b98034f82bf1..ad0f8a997e48 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -309,7 +309,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=4) + parser.add_argument('--batch_size', type=int, default=8) args = parser.parse_args() # check if saved model if path.exists("gnn_llm.pt"): From 9409c80c0d0720bb4c0498cf8e3d8c2f87907d45 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:01:16 -0700 Subject: [PATCH 276/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ad0f8a997e48..d38f2b8b9941 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -107,9 +107,9 @@ def adjust_learning_rate(param_group, LR, epoch): train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, pin_memory=True, shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=batch_size, + val_loader = DataLoader(val_dataset, batch_size=2 * batch_size, drop_last=False, pin_memory=True, shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=batch_size, + test_loader = DataLoader(test_dataset, batch_size=2 * batch_size, drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model From c1b4d3b620877ba755a27be89a89cdaa8a4c3454 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:24:11 -0700 Subject: [PATCH 277/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d38f2b8b9941..5c58a3c2eab1 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -309,7 +309,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=8) + parser.add_argument('--batch_size', type=int, default=32) args = parser.parse_args() # check if saved model if path.exists("gnn_llm.pt"): From 28519f3546ad3dde58f4198bf503329c024e1dbc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:31:15 -0700 Subject: [PATCH 278/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5c58a3c2eab1..16fd67716290 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -306,10 +306,10 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--hidden_channels', type=int, default=1024) - parser.add_argument('--num_gnn_layers', type=int, default=4) + parser.add_argument('--num_gnn_layers', type=int, default=3) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--batch_size', type=int, default=4) args = parser.parse_args() # check if saved model if path.exists("gnn_llm.pt"): From 68b96368d419a32601996c2d634ca20fffab3a74 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:31:31 -0700 Subject: [PATCH 279/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 16fd67716290..525a1c401a8b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -334,7 +334,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - print("Here is a minimal demo showcasing how \ - GNN+LLM can solve LLM hallucinations?") - print("First comparing against a pretrained LLAMA2 model") - minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size) + # print("Here is a minimal demo showcasing how \ + # GNN+LLM can solve LLM hallucinations?") + # print("First comparing against a pretrained LLAMA2 model") + # minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size) From c00ea5dd362435ff21445b2cc12457446da5b911 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:39:47 -0700 Subject: [PATCH 280/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 525a1c401a8b..90ed10a39d2a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -312,13 +312,14 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): parser.add_argument('--batch_size', type=int, default=4) args = parser.parse_args() # check if saved model - if path.exists("gnn_llm.pt"): - print("Existing trained model found.") - print("Would you like to retrain?") - user_input = str(input("(y/n):")).lower() - retrain = user_input == "y" - else: - retrain = True + retrain = True + # if path.exists("gnn_llm.pt"): + # print("Existing trained model found.") + # print("Would you like to retrain?") + # user_input = str(input("(y/n):")).lower() + # retrain = user_input == "y" + # else: + # retrain = True if retrain: since = time.time() prep_time, dataset, model = train(since, args.epochs, @@ -330,7 +331,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): gc.collect() e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") - print("E2E time minus Prep Time =", e2e_time - prep_time, "seconds") + print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() From 69c5914b75ee13d3411a7040d547c1ee970a74df Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 10:47:35 -0700 Subject: [PATCH 281/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 90ed10a39d2a..d2c9d02f18ed 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -306,7 +306,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--hidden_channels', type=int, default=1024) - parser.add_argument('--num_gnn_layers', type=int, default=3) + parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) parser.add_argument('--batch_size', type=int, default=4) From b3f7b8e5d13a5ac9dc801c7d0aa432f880e67938 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 11:05:12 -0700 Subject: [PATCH 282/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d2c9d02f18ed..56e933a62563 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -116,6 +116,10 @@ def adjust_learning_rate(param_group, LR, epoch): if model is None: model = GNN_LLM(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) + if num_gnn_layers is not None: + model_save_name = "gnn_llm" + else: + model_save_name = "llm" # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -133,6 +137,7 @@ def adjust_learning_rate(param_group, LR, epoch): all params: {all_param} || \ trainable%: {100 * trainable_params / all_param}") + best_val_loss = float('inf') # Step 4 Training for epoch in range(num_epochs): model.train() @@ -171,12 +176,16 @@ def adjust_learning_rate(param_group, LR, epoch): val_loss += loss.item() val_loss = val_loss / len(val_loader) print(epoch_str + f", Val Loss: {val_loss}") + if val_loss < best_val_loss: + best_val_loss = val_loss + torch.save(model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() # Step 5 Evaluating print("Final Evaluation...") + model = torch.load(model_save_name + "_best_val_loss_ckpt.pt") model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) @@ -192,10 +201,7 @@ def adjust_learning_rate(param_group, LR, epoch): print(f'Test Acc {acc}') # save model print("Saving Model...") - if num_gnn_layers is not None: - torch.save(model, "gnn_llm.pt") - else: - torch.save(model, "llm.pt") + torch.save(model_save_name + ".pt") print("Done!") return prep_time, dataset, model From 6a9537b5afc7f5fdd31566c354c77e973db89254 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 11:56:44 -0700 Subject: [PATCH 283/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 56e933a62563..cb77209b18af 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -178,7 +178,7 @@ def adjust_learning_rate(param_group, LR, epoch): print(epoch_str + f", Val Loss: {val_loss}") if val_loss < best_val_loss: best_val_loss = val_loss - torch.save(model_save_name + "_best_val_loss_ckpt.pt") + torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From a6b9bb4022a5ea3734f47303f6d362191d301a61 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 12:40:20 -0700 Subject: [PATCH 284/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index cb77209b18af..cb22c03f6765 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -201,7 +201,7 @@ def adjust_learning_rate(param_group, LR, epoch): print(f'Test Acc {acc}') # save model print("Saving Model...") - torch.save(model_save_name + ".pt") + torch.save(model, model_save_name + ".pt") print("Done!") return prep_time, dataset, model From 4e58b4748b8848e91600a6e8a7ace4fa9a440c40 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 13:20:51 -0700 Subject: [PATCH 285/752] debugging --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index cb22c03f6765..fcb39bb3aba3 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -177,6 +177,7 @@ def adjust_learning_rate(param_group, LR, epoch): val_loss = val_loss / len(val_loader) print(epoch_str + f", Val Loss: {val_loss}") if val_loss < best_val_loss: + print("Checkpointing best val loss model...") best_val_loss = val_loss torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") From 1df9c2c65a02133f3b1a00b41880b8a5756c6bd7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 13:29:00 -0700 Subject: [PATCH 286/752] trying w/o LORA --- examples/llm_plus_gnn/g_retriever.py | 2 +- torch_geometric/nn/models/gnn_llm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index fcb39bb3aba3..8d3eddeff215 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -316,7 +316,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=4) + parser.add_argument('--batch_size', type=int, default=8) args = parser.parse_args() # check if saved model retrain = True diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a102e2cb13e2..32ab01ada80a 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -245,7 +245,7 @@ class GNN_LLM(nn.Module): def __init__( self, llm_to_use='llama2', - llm_use_lora: bool = True, + llm_use_lora: bool = False, llm_dtype=torch.bfloat16, num_llm_params: int = 7, gnn_to_use=GAT, From f7667d29ed91c1072f6955d75f978a08ace40118 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 13:43:25 -0700 Subject: [PATCH 287/752] trying w/o LORA --- examples/llm_plus_gnn/g_retriever.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 8d3eddeff215..2da090d6a8b9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -80,7 +80,7 @@ def compute_accuracy(eval_output): return hit -def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, lr, +def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, model=None, dataset=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup @@ -107,9 +107,9 @@ def adjust_learning_rate(param_group, LR, epoch): train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, pin_memory=True, shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=2 * batch_size, + val_loader = DataLoader(val_dataset, batch_size=eval_batch_size, drop_last=False, pin_memory=True, shuffle=False) - test_loader = DataLoader(test_dataset, batch_size=2 * batch_size, + test_loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, pin_memory=True, shuffle=False) # Step 2: Build Model @@ -207,7 +207,7 @@ def adjust_learning_rate(param_group, LR, epoch): return prep_time, dataset, model -def minimal_demo(model, dataset, lr, epochs, batch_size): +def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): # Step 1: Define a single batch size test loader idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] @@ -271,7 +271,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") - _, _, pure_llm = train(since, 1, None, None, batch_size, lr, + _, _, pure_llm = train(since, 1, None, None, batch_size, eval_batch_size, lr, model=pure_llm, dataset=dataset) print("E2E time (e2e_time) =", e2e_time, "seconds") else: @@ -316,7 +316,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--batch_size', type=int, default=8) + parser.add_argument('--train_batch_size', type=int, default=8) + parser.add_argument('--eval_batch_size', type=int, default=16) + args = parser.parse_args() # check if saved model retrain = True @@ -332,7 +334,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): prep_time, dataset, model = train(since, args.epochs, args.hidden_channels, args.num_gnn_layers, args.batch_size, - args.lr) + args.eval_batch_size, args.lr) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -345,4 +347,4 @@ def minimal_demo(model, dataset, lr, epochs, batch_size): # print("Here is a minimal demo showcasing how \ # GNN+LLM can solve LLM hallucinations?") # print("First comparing against a pretrained LLAMA2 model") - # minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size) + # minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) From 340d97999c59c1cf0629b41f9063461d41b2a475 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 20:44:31 +0000 Subject: [PATCH 288/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2da090d6a8b9..83c076477558 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -80,8 +80,8 @@ def compute_accuracy(eval_output): return hit -def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, - model=None, dataset=None): +def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, + eval_batch_size, lr, model=None, dataset=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -271,8 +271,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") - _, _, pure_llm = train(since, 1, None, None, batch_size, eval_batch_size, lr, - model=pure_llm, dataset=dataset) + _, _, pure_llm = train(since, 1, None, None, batch_size, + eval_batch_size, lr, model=pure_llm, + dataset=dataset) print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm = torch.save("llm.pt") From 31ee2921d313ead5677362c7730f59169a82e0d4 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 13:47:20 -0700 Subject: [PATCH 289/752] trying w/o LORA --- examples/llm_plus_gnn/g_retriever.py | 86 +++++++++++++++------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2da090d6a8b9..20c221ec2313 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -139,47 +139,51 @@ def adjust_learning_rate(param_group, LR, epoch): best_val_loss = float('inf') # Step 4 Training - for epoch in range(num_epochs): - model.train() - epoch_loss = 0. - if epoch == 0: - prep_time = round(time.time() - since, 2) - print("Total Prep Time (prep_time) =", prep_time) - print("Training beginning...") - epoch_str = f"Epoch: {epoch + 1}|{num_epochs}" - loader = tqdm(train_loader, desc=epoch_str) - for step, batch in enumerate(loader): - optimizer.zero_grad() - loss = model(batch) - loss.backward() - - clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) - - if (step + 1) % grad_steps == 0: - adjust_learning_rate(optimizer.param_groups[0], lr, - step / len(train_loader) + epoch) - - optimizer.step() - epoch_loss = epoch_loss + loss.item() - - if (step + 1) % grad_steps == 0: - lr = optimizer.param_groups[0]["lr"] - train_loss = epoch_loss / len(train_loader) - print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") - - val_loss = 0. - eval_output = [] - model.eval() - with torch.no_grad(): - for step, batch in enumerate(val_loader): + try: + for epoch in range(num_epochs): + model.train() + epoch_loss = 0. + if epoch == 0: + prep_time = round(time.time() - since, 2) + print("Total Prep Time (prep_time) =", prep_time) + print("Training beginning...") + epoch_str = f"Epoch: {epoch + 1}|{num_epochs}" + loader = tqdm(train_loader, desc=epoch_str) + for step, batch in enumerate(loader): + optimizer.zero_grad() loss = model(batch) - val_loss += loss.item() - val_loss = val_loss / len(val_loader) - print(epoch_str + f", Val Loss: {val_loss}") - if val_loss < best_val_loss: - print("Checkpointing best val loss model...") - best_val_loss = val_loss - torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") + loss.backward() + + clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) + + if (step + 1) % grad_steps == 0: + adjust_learning_rate(optimizer.param_groups[0], lr, + step / len(train_loader) + epoch) + + optimizer.step() + epoch_loss = epoch_loss + loss.item() + + if (step + 1) % grad_steps == 0: + lr = optimizer.param_groups[0]["lr"] + train_loss = epoch_loss / len(train_loader) + print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") + + val_loss = 0. + eval_output = [] + model.eval() + with torch.no_grad(): + for step, batch in enumerate(val_loader): + loss = model(batch) + val_loss += loss.item() + val_loss = val_loss / len(val_loader) + print(epoch_str + f", Val Loss: {val_loss}") + if val_loss < best_val_loss: + print("Checkpointing best val loss model...") + best_val_loss = val_loss + torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") + except: # noqa + # allows to ctrl-C in training and still eval with the best checkpoint + pass torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() @@ -316,7 +320,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) parser.add_argument('--epochs', type=int, default=10) - parser.add_argument('--train_batch_size', type=int, default=8) + parser.add_argument('--batch_size', type=int, default=8) parser.add_argument('--eval_batch_size', type=int, default=16) args = parser.parse_args() From c0a0cce10784d795e036bf03e053cd7f5f389157 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 20:48:39 +0000 Subject: [PATCH 290/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2d28cba75d9e..7473c0675164 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -181,7 +181,7 @@ def adjust_learning_rate(param_group, LR, epoch): print("Checkpointing best val loss model...") best_val_loss = val_loss torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") - except: # noqa + except: # noqa # allows to ctrl-C in training and still eval with the best checkpoint pass From 86c555ae9ba7e7b8c70758e5a4ced890fc4fef6b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 13:59:22 -0700 Subject: [PATCH 291/752] trying w/o LORAccccbldnentdgedjttcerditubgctrdlecvtkifvukgc --- examples/llm_plus_gnn/g_retriever.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2d28cba75d9e..c2fe0de9cd2b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -81,7 +81,7 @@ def compute_accuracy(eval_output): def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, - eval_batch_size, lr, model=None, dataset=None): + eval_batch_size, lr, model=None, dataset=None, checkpointing=False): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -177,7 +177,7 @@ def adjust_learning_rate(param_group, LR, epoch): val_loss += loss.item() val_loss = val_loss / len(val_loader) print(epoch_str + f", Val Loss: {val_loss}") - if val_loss < best_val_loss: + if checkpointing and val_loss < best_val_loss: print("Checkpointing best val loss model...") best_val_loss = val_loss torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") @@ -190,7 +190,8 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 5 Evaluating print("Final Evaluation...") - model = torch.load(model_save_name + "_best_val_loss_ckpt.pt") + if checkpointing: + model = torch.load(model_save_name + "_best_val_loss_ckpt.pt") model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) @@ -317,12 +318,15 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--hidden_channels', type=int, default=1024) + parser.add_argument('--gnn_hidden_channels', type=int, default=1024) parser.add_argument('--num_gnn_layers', type=int, default=4) parser.add_argument('--lr', type=float, default=1e-5) - parser.add_argument('--epochs', type=int, default=10) + parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch_size', type=int, default=8) parser.add_argument('--eval_batch_size', type=int, default=16) + parser.add_argument("--checkpointing", type=bool, action="store_true", + help="Use this flag to checkpoint each time a \ + new best val loss is achieved") args = parser.parse_args() # check if saved model @@ -337,9 +341,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): if retrain: since = time.time() prep_time, dataset, model = train(since, args.epochs, - args.hidden_channels, + args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, - args.eval_batch_size, args.lr) + args.eval_batch_size, args.lr, args.checkpointing) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 364e4de608343985847cb2b72669668146ae6a0e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 14:01:37 -0700 Subject: [PATCH 292/752] seems turning off LORA fixes the issue and makes training much faster --- examples/llm_plus_gnn/g_retriever.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1bbbb59a1673..41c9481c4d83 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -331,13 +331,13 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): args = parser.parse_args() # check if saved model retrain = True - # if path.exists("gnn_llm.pt"): - # print("Existing trained model found.") - # print("Would you like to retrain?") - # user_input = str(input("(y/n):")).lower() - # retrain = user_input == "y" - # else: - # retrain = True + if path.exists("gnn_llm.pt"): + print("Existing trained model found.") + print("Would you like to retrain?") + user_input = str(input("(y/n):")).lower() + retrain = user_input == "y" + else: + retrain = True if retrain: since = time.time() prep_time, dataset, model = train(since, args.epochs, @@ -353,7 +353,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - # print("Here is a minimal demo showcasing how \ - # GNN+LLM can solve LLM hallucinations?") - # print("First comparing against a pretrained LLAMA2 model") - # minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) + print("Here is a minimal demo showcasing how \ + GNN+LLM can solve LLM hallucinations?") + print("First comparing against a pretrained LLAMA2 model") + minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) From a1fe2842c9dc0172e77705d77c130d13f477b23a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:02:18 +0000 Subject: [PATCH 293/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 41c9481c4d83..53925297d23a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -324,7 +324,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch_size', type=int, default=8) parser.add_argument('--eval_batch_size', type=int, default=16) - parser.add_argument("--checkpointing", type=bool, action="store_true", + parser.add_argument( + "--checkpointing", type=bool, action="store_true", help="Use this flag to checkpoint each time a \ new best val loss is achieved") @@ -343,7 +344,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): prep_time, dataset, model = train(since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, - args.eval_batch_size, args.lr, args.checkpointing) + args.eval_batch_size, args.lr, + args.checkpointing) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 2ab810db223fa28945c2a5656a6c37296e13db6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:03:22 +0000 Subject: [PATCH 294/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 53925297d23a..8470a7d82711 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -358,4 +358,5 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): print("Here is a minimal demo showcasing how \ GNN+LLM can solve LLM hallucinations?") print("First comparing against a pretrained LLAMA2 model") - minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) + minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, + args.eval_batch_size) From d27c55963afc0c07c9da576f96145a5df6c2adf2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 14:06:39 -0700 Subject: [PATCH 295/752] seems turning off LORA fixes the issue and makes training much faster --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 41c9481c4d83..a15e37600d5f 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -324,7 +324,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch_size', type=int, default=8) parser.add_argument('--eval_batch_size', type=int, default=16) - parser.add_argument("--checkpointing", type=bool, action="store_true", + parser.add_argument("--checkpointing", action="store_true", help="Use this flag to checkpoint each time a \ new best val loss is achieved") From dc8867aa8a28cdae697c455794359491d99a8e2e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:08:48 +0000 Subject: [PATCH 296/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 4ae4aabc3115..c5f0ddf451cf 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -324,7 +324,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch_size', type=int, default=8) parser.add_argument('--eval_batch_size', type=int, default=16) - parser.add_argument("--checkpointing", action="store_true", + parser.add_argument( + "--checkpointing", action="store_true", help="Use this flag to checkpoint each time a \ new best val loss is achieved") From 3e3f79ee372a5d6bbbe69ad2f70eaad3acf476c6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 14:10:13 -0700 Subject: [PATCH 297/752] seems turning off LORA fixes the issue and makes training much faster --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 4ae4aabc3115..178fee5f3d90 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -344,7 +344,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, args.eval_batch_size, args.lr, - args.checkpointing) + checkpointing=args.checkpointing) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 1f64d39eb01df44643d9401382e81e6c414fe223 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 14:26:38 -0700 Subject: [PATCH 298/752] seems turning off LORA fixes the issue and makes training much faster --- examples/llm_plus_gnn/g_retriever.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ac4d2ca6ed7b..7e86ce9eb946 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -247,21 +247,12 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): continue gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) pure_llm_hallucin_sum += int(pure_llm_hallucinates) - # showcase LLM hallucinations solved by GNN - if pure_llm_hallucinates and not gnn_llm_hallucinates: - final_prnt_str += "Prompt: " + question + "\n" - final_prnt_str += "Label: " + correct_answer + "\n" - final_prnt_str += "Pure LLM Output: " + pure_llm_pred + "\n" - final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" - final_prnt_str += "#" * 20 + "\n" print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) percent = 100.0 * round(1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) print(f"GNN reduces hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") - print("Potential instances where GNN solves the hallucinations of LLM") - print(final_prnt_str) print("Now we see how the LLM compares when finetuned...") since = time.time() trained_hallucin_sum = 0 @@ -304,8 +295,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" print("After finetuning the LLM...") - print("Total Tuned LLM Hallucinations:", untuned_llm_hallucin_sum) - print("Total Tuned LLM Hallucinations:", trained_llm_hallucin_sum) + print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) + print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") tuned_percent = 100.0 * round( From d0b91aa31dd67f6dddbde4c354371c4dd9d99040 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 15:02:01 -0700 Subject: [PATCH 299/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 7e86ce9eb946..e76232f576fc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -346,8 +346,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): else: model = torch.load("gnn_llm.pt") dataset = WebQSPDataset() - print("Here is a minimal demo showcasing how \ - GNN+LLM can solve LLM hallucinations?") + print("Here's a demo showcasing how GNN reduces LLM hallucinations:") print("First comparing against a pretrained LLAMA2 model") minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) From 1d25cb7c9e37c2d5a6fb0ff82ee3ac1f9aea0f20 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 15:06:56 -0700 Subject: [PATCH 300/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index e76232f576fc..95033cfbf8e2 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -4,7 +4,7 @@ by 54% compared to the [LLM] baseline“. requirements on top of basic PyG: -pip install peft datasets transformers pcst_fast sentencepiece tqdm pandas +pip install datasets transformers pcst_fast sentencepiece tqdm pandas """ import argparse import gc From bd3d60621e7f447ccadd55f493bd7ef2c0f443c5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 12 Apr 2024 16:20:21 -0700 Subject: [PATCH 301/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 95033cfbf8e2..edcb76f54381 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -258,6 +258,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): trained_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" + del pure_llm, model if path.exists("llm.pt"): print("Existing finetuned LLAMA2 found.") print("Would you like to retrain?") From 74fa2dc2faa6e81c02fcee4c5ff01b7a09bbda3d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 11:10:12 -0700 Subject: [PATCH 302/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 85 +++++++++++++--------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index edcb76f54381..007fc91d42bf 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -139,52 +139,47 @@ def adjust_learning_rate(param_group, LR, epoch): best_val_loss = float('inf') # Step 4 Training - try: - for epoch in range(num_epochs): - model.train() - epoch_loss = 0. - if epoch == 0: - prep_time = round(time.time() - since, 2) - print("Total Prep Time (prep_time) =", prep_time) - print("Training beginning...") - epoch_str = f"Epoch: {epoch + 1}|{num_epochs}" - loader = tqdm(train_loader, desc=epoch_str) - for step, batch in enumerate(loader): - optimizer.zero_grad() + for epoch in range(num_epochs): + model.train() + epoch_loss = 0. + if epoch == 0: + prep_time = round(time.time() - since, 2) + print("Total Prep Time (prep_time) =", prep_time) + print("Training beginning...") + epoch_str = f"Epoch: {epoch + 1}|{num_epochs}" + loader = tqdm(train_loader, desc=epoch_str) + for step, batch in enumerate(loader): + optimizer.zero_grad() + loss = model(batch) + loss.backward() + + clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) + + if (step + 1) % grad_steps == 0: + adjust_learning_rate(optimizer.param_groups[0], lr, + step / len(train_loader) + epoch) + + optimizer.step() + epoch_loss = epoch_loss + loss.item() + + if (step + 1) % grad_steps == 0: + lr = optimizer.param_groups[0]["lr"] + train_loss = epoch_loss / len(train_loader) + print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") + + val_loss = 0. + eval_output = [] + model.eval() + with torch.no_grad(): + for step, batch in enumerate(val_loader): loss = model(batch) - loss.backward() - - clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) - - if (step + 1) % grad_steps == 0: - adjust_learning_rate(optimizer.param_groups[0], lr, - step / len(train_loader) + epoch) - - optimizer.step() - epoch_loss = epoch_loss + loss.item() - - if (step + 1) % grad_steps == 0: - lr = optimizer.param_groups[0]["lr"] - train_loss = epoch_loss / len(train_loader) - print(epoch_str + f",Train Loss (Epoch Mean): {train_loss}") - - val_loss = 0. - eval_output = [] - model.eval() - with torch.no_grad(): - for step, batch in enumerate(val_loader): - loss = model(batch) - val_loss += loss.item() - val_loss = val_loss / len(val_loader) - print(epoch_str + f", Val Loss: {val_loss}") - if checkpointing and val_loss < best_val_loss: - print("Checkpointing best val loss model...") - best_val_loss = val_loss - torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") - except: # noqa - # allows to ctrl-C in training and still eval with the best checkpoint - pass - + val_loss += loss.item() + val_loss = val_loss / len(val_loader) + print(epoch_str + f", Val Loss: {val_loss}") + if checkpointing and val_loss < best_val_loss: + print("Checkpointing best val loss model...") + best_val_loss = val_loss + torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From 6ebba50be38b1f12455077265a70811df770ac12 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 13:18:32 -0700 Subject: [PATCH 303/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 007fc91d42bf..d2b2e8b6cd64 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -253,7 +253,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): trained_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" - del pure_llm, model + del model if path.exists("llm.pt"): print("Existing finetuned LLAMA2 found.") print("Would you like to retrain?") From ee5b40aab5808804b597334e5c0edb88898acef1 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 15:32:13 -0700 Subject: [PATCH 304/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 68 +++++++++++++++++----------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d2b2e8b6cd64..ac46b1242eb9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -216,7 +216,13 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pin_memory=True, shuffle=False) # define the pure pretrained LLM pure_llm = LLM() - + if path.exists("gnn_llm_demo_outs.txt") and path.exists("untuned_llm_demo_outs.txt"): + print("Demo outputs for LLM and GNN+LLM found.") + print("Woudl you like to reuse them?") + user_input = str(input("(y/n):")).lower() + redemo = user_input == "y" + else: + redemo = True # Step loop through the loader and run both models gnn_llm_hallucin_sum = 0 pure_llm_hallucin_sum = 0 @@ -224,32 +230,38 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") gnn_save_list = [] untuned_llm_save_list = [] - for batch in tqdm(loader): - question = batch.question[0] - correct_answer = batch.label[0] - gnn_llm_out = model.inference(batch) - # GNN+LLM only using 32 tokens to answer, give untrained LLM more - pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) - gnn_llm_pred = gnn_llm_out['pred'][0] - pure_llm_pred = pure_llm_out['pred'][0] - gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] - pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, - correct_answer) - untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] - if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": - # skipping since hard to evaluate if the answer is a hallucination - continue - gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) - pure_llm_hallucin_sum += int(pure_llm_hallucinates) - print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) - print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) - percent = 100.0 * round(1 - - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) - print(f"GNN reduces hallucinations by: ~{percent}%") - print("Note: hallucinations detected by regex hence the ~") - print("Now we see how the LLM compares when finetuned...") - since = time.time() + if redemo: + for batch in tqdm(loader): + question = batch.question[0] + correct_answer = batch.label[0] + gnn_llm_out = model.inference(batch) + # GNN+LLM only using 32 tokens to answer, give untrained LLM more + pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) + gnn_llm_pred = gnn_llm_out['pred'][0] + pure_llm_pred = pure_llm_out['pred'][0] + gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) + gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] + pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, + correct_answer) + untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": + # skipping since hard to evaluate if the answer is a hallucination + continue + gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) + pure_llm_hallucin_sum += int(pure_llm_hallucinates) + print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) + percent = 100.0 * round(1 - + (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) + print(f"GNN reduces hallucinations by: ~{percent}%") + print("Note: hallucinations detected by regex hence the ~") + print("Now we see how the LLM compares when finetuned...") + print("Saving outputs of GNN+LLM and Pretrained LLM...") + torch.save(gnn_save_list, "gnn_llm_demo_outs.txt") + torch.save(untuned_llm_save_list, "untuned_llm_demo_outs.txt") + else: + gnn_save_list = torch.load("gnn_llm_demo_outs.txt") + untuned_llm_save_list = torch.load("untuned_llm_demo_outs.txt") trained_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" @@ -263,9 +275,11 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): retrain = True if retrain: print("Finetuning LLAMA2...") + since = time.time() _, _, pure_llm = train(since, 1, None, None, batch_size, eval_batch_size, lr, model=pure_llm, dataset=dataset) + e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm = torch.save("llm.pt") From 898931df43021eb470c3107fee2a860bea3eb9fb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:33:20 +0000 Subject: [PATCH 305/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ac46b1242eb9..65e926d32506 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -216,7 +216,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pin_memory=True, shuffle=False) # define the pure pretrained LLM pure_llm = LLM() - if path.exists("gnn_llm_demo_outs.txt") and path.exists("untuned_llm_demo_outs.txt"): + if path.exists("gnn_llm_demo_outs.txt") and path.exists( + "untuned_llm_demo_outs.txt"): print("Demo outputs for LLM and GNN+LLM found.") print("Woudl you like to reuse them?") user_input = str(input("(y/n):")).lower() @@ -239,7 +240,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) gnn_llm_pred = gnn_llm_out['pred'][0] pure_llm_pred = pure_llm_out['pred'][0] - gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) + gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, + correct_answer) gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) @@ -251,8 +253,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_hallucin_sum += int(pure_llm_hallucinates) print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) - percent = 100.0 * round(1 - - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) + percent = 100.0 * round( + 1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) print(f"GNN reduces hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") print("Now we see how the LLM compares when finetuned...") From 0abd3ba8ebd5520adce97de8ee43a3431f821c11 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 16:12:35 -0700 Subject: [PATCH 306/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ac46b1242eb9..7533f2d16391 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -217,8 +217,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): # define the pure pretrained LLM pure_llm = LLM() if path.exists("gnn_llm_demo_outs.txt") and path.exists("untuned_llm_demo_outs.txt"): - print("Demo outputs for LLM and GNN+LLM found.") - print("Woudl you like to reuse them?") + print("Saved demo outputs for LLM and GNN+LLM found.") + print("Would you like to reuse them?") user_input = str(input("(y/n):")).lower() redemo = user_input == "y" else: @@ -257,11 +257,20 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): print("Note: hallucinations detected by regex hence the ~") print("Now we see how the LLM compares when finetuned...") print("Saving outputs of GNN+LLM and Pretrained LLM...") - torch.save(gnn_save_list, "gnn_llm_demo_outs.txt") - torch.save(untuned_llm_save_list, "untuned_llm_demo_outs.txt") + save_dict = { + "gnn_save_list": gnn_save_list, + "untuned_llm_save_list": untuned_llm_save_list, + "gnn_hallucin_sum":gnn_hallucin_sum, + "pure_llm_hallucin_sum":pure_llm_hallucin_sum + } + torch.save(save_dict, "demo_save_dict.pt") else: - gnn_save_list = torch.load("gnn_llm_demo_outs.txt") - untuned_llm_save_list = torch.load("untuned_llm_demo_outs.txt") + save_dict = torch.load("demo_save_dict.pt") + gnn_save_list = save_dict["gnn_save_list"] + untuned_llm_save_list = save_dict["untuned_llm_save_list"] + gnn_hallucin_sum = save_dict["gnn_hallucin_sum"] + pure_llm_hallucin_sum = save_dict["pure_llm_hallucin_sum"] + trained_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" From 0a7d44be099639f01b59c2b467c3bc37e9da326f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 16:15:28 -0700 Subject: [PATCH 307/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index a8b2e1e02373..532618983a63 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -216,28 +216,20 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pin_memory=True, shuffle=False) # define the pure pretrained LLM pure_llm = LLM() -<<<<<<< HEAD - if path.exists("gnn_llm_demo_outs.txt") and path.exists("untuned_llm_demo_outs.txt"): + if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") print("Would you like to reuse them?") -======= - if path.exists("gnn_llm_demo_outs.txt") and path.exists( - "untuned_llm_demo_outs.txt"): - print("Demo outputs for LLM and GNN+LLM found.") - print("Woudl you like to reuse them?") ->>>>>>> 898931df43021eb470c3107fee2a860bea3eb9fb user_input = str(input("(y/n):")).lower() redemo = user_input == "y" else: redemo = True - # Step loop through the loader and run both models - gnn_llm_hallucin_sum = 0 - pure_llm_hallucin_sum = 0 - final_prnt_str = "" - print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") - gnn_save_list = [] - untuned_llm_save_list = [] if redemo: + # Step loop through the loader and run both models + print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") + gnn_llm_hallucin_sum = 0 + pure_llm_hallucin_sum = 0 + gnn_save_list = [] + untuned_llm_save_list = [] for batch in tqdm(loader): question = batch.question[0] correct_answer = batch.label[0] From e94a4f7761ce1540118fc68d96f5735c42e45d57 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 23:16:32 +0000 Subject: [PATCH 308/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 532618983a63..e67b1190b566 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -225,7 +225,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): redemo = True if redemo: # Step loop through the loader and run both models - print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") + print( + "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") gnn_llm_hallucin_sum = 0 pure_llm_hallucin_sum = 0 gnn_save_list = [] @@ -260,8 +261,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): save_dict = { "gnn_save_list": gnn_save_list, "untuned_llm_save_list": untuned_llm_save_list, - "gnn_hallucin_sum":gnn_hallucin_sum, - "pure_llm_hallucin_sum":pure_llm_hallucin_sum + "gnn_hallucin_sum": gnn_hallucin_sum, + "pure_llm_hallucin_sum": pure_llm_hallucin_sum } torch.save(save_dict, "demo_save_dict.pt") else: From f09bab37c9cfa974b4dc9b2e6b6c762d0de4afa6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 16:50:52 -0700 Subject: [PATCH 309/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 532618983a63..fc6352afe7fb 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -244,8 +244,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] - if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": - # skipping since hard to evaluate if the answer is a hallucination + if gnn_llm_hallucinates == "skip" or \ + pure_llm_hallucinates == "skip": + # skipping when hallucination is hard to eval continue gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) pure_llm_hallucin_sum += int(pure_llm_hallucinates) @@ -260,7 +261,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): save_dict = { "gnn_save_list": gnn_save_list, "untuned_llm_save_list": untuned_llm_save_list, - "gnn_hallucin_sum":gnn_hallucin_sum, + "gnn_llm_hallucin_sum":gnn_llm_hallucin_sum, "pure_llm_hallucin_sum":pure_llm_hallucin_sum } torch.save(save_dict, "demo_save_dict.pt") @@ -268,10 +269,10 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): save_dict = torch.load("demo_save_dict.pt") gnn_save_list = save_dict["gnn_save_list"] untuned_llm_save_list = save_dict["untuned_llm_save_list"] - gnn_hallucin_sum = save_dict["gnn_hallucin_sum"] + gnn_llm_hallucin_sum = save_dict["gnn_llm_hallucin_sum"] pure_llm_hallucin_sum = save_dict["pure_llm_hallucin_sum"] - trained_hallucin_sum = 0 + trained_llm_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" del model @@ -293,12 +294,13 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): else: pure_llm = torch.save("llm.pt") print("Evaluating Tuned LLM...") - for batch in tqdm(enumerate(loader)): + for i, batch in tqdm(enumerate(loader)): question = batch.question[0] correct_answer = batch.label[0] gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] - if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": + if gnn_llm_hallucinates == "skip" or \ + untuned_llm_hallucinates == "skip": continue pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, From c1779bfe710a424b2a49c57d633089563f92928d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 23:52:26 +0000 Subject: [PATCH 310/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 8e708f7d9621..6d37648c722d 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -262,8 +262,8 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): save_dict = { "gnn_save_list": gnn_save_list, "untuned_llm_save_list": untuned_llm_save_list, - "gnn_llm_hallucin_sum":gnn_llm_hallucin_sum, - "pure_llm_hallucin_sum":pure_llm_hallucin_sum + "gnn_llm_hallucin_sum": gnn_llm_hallucin_sum, + "pure_llm_hallucin_sum": pure_llm_hallucin_sum } torch.save(save_dict, "demo_save_dict.pt") else: From 2c1c89ba59fe71c8aad64294a02e996eeffdf2a8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 17:01:00 -0700 Subject: [PATCH 311/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 8e708f7d9621..391216d5065a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -245,8 +245,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] - if gnn_llm_hallucinates == "skip" or \ - pure_llm_hallucinates == "skip": + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa # skipping when hallucination is hard to eval continue gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) @@ -300,8 +299,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): correct_answer = batch.label[0] gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] - if gnn_llm_hallucinates == "skip" or \ - untuned_llm_hallucinates == "skip": + if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, From d6d2b2f971ff5c5ea6a88e3a6d9934707a30ed18 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 00:02:19 +0000 Subject: [PATCH 312/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0bfc278f9f00..48f932849e81 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -245,7 +245,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] - if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa # skipping when hallucination is hard to eval continue gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) @@ -299,7 +299,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): correct_answer = batch.label[0] gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] - if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa + if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue pure_llm_pred = pure_llm.inference(batch)['pred'][0] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, From 248b9dfe93484fde614f85197f56b1f5eef0fb64 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 18:45:17 -0700 Subject: [PATCH 313/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0bfc278f9f00..d14c1d71eacd 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -292,7 +292,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: - pure_llm = torch.save("llm.pt") + pure_llm = torch.load("llm.pt") print("Evaluating Tuned LLM...") for i, batch in tqdm(enumerate(loader)): question = batch.question[0] From 64e542f241aaaaf94b4c79a2b471e9ba8757000b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 18:47:24 -0700 Subject: [PATCH 314/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0ceade1a29a6..642559c00872 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -220,10 +220,10 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): print("Saved demo outputs for LLM and GNN+LLM found.") print("Would you like to reuse them?") user_input = str(input("(y/n):")).lower() - redemo = user_input == "y" + skip_step_one = user_input == "y" else: - redemo = True - if redemo: + skip_step_one = False + if not skip_step_one: # Step loop through the loader and run both models print( "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") From 0e328c463b3726a31420586d205d440f481204de Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 18:51:25 -0700 Subject: [PATCH 315/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 642559c00872..aee196c46a59 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -241,10 +241,10 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [gnn_llm_pred, gnn_llm_hallucinates] + gnn_save_list += [(gnn_llm_pred, gnn_llm_hallucinates)] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) - untuned_llm_save_list += [pure_llm_pred, pure_llm_hallucinates] + untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa # skipping when hallucination is hard to eval continue From 1b5c9942e793c596bc7d9d01176ec4046bafd321 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 15 Apr 2024 19:35:59 -0700 Subject: [PATCH 316/752] cleanup --- torch_geometric/nn/models/gnn_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 32ab01ada80a..a66a4a57aae1 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -73,7 +73,7 @@ def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, else: self.printable_llm_name = llm_name self.huggingface_str = llm_name - self.mem_needed = 75 * num_params / 7 + self.mem_needed = 85 * num_params / 7 self.llm_dtype = llm_dtype print('Loading ' + str(self.printable_llm_name)) kwargs = get_llm_kwargs(self.mem_needed) From 6e094fb7465592ad42258232b25753695071e27c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 09:43:49 -0700 Subject: [PATCH 317/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 43 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index aee196c46a59..6d86d5ed3705 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -194,7 +194,6 @@ def adjust_learning_rate(param_group, LR, epoch): with torch.no_grad(): output = model.inference(batch) eval_output.append(output) - progress_bar_test.update(1) # Step 6 Post-processing & compute metrics @@ -203,18 +202,13 @@ def adjust_learning_rate(param_group, LR, epoch): # save model print("Saving Model...") torch.save(model, model_save_name + ".pt") + print("Saving eval output for downstream demo...") + torch.save(eval_output, model_save_name + "_eval_outs.pt") print("Done!") - return prep_time, dataset, model + return prep_time, dataset, eval_output -def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): - # Step 1: Define a single batch size test loader - idx_split = dataset.split_idxs - test_dataset = [dataset[i] for i in idx_split['test']] - # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=1, drop_last=False, - pin_memory=True, shuffle=False) - # define the pure pretrained LLM +def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") @@ -223,7 +217,14 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): skip_step_one = user_input == "y" else: skip_step_one = False + if not skip_step_one: + # Step 1: Define a single batch size test loader + idx_split = dataset.split_idxs + test_dataset = [dataset[i] for i in idx_split['test']] + # batch size 1 loader for simplicity + loader = DataLoader(test_dataset, batch_size=1, drop_last=False, + pin_memory=True, shuffle=False) # Step loop through the loader and run both models print( "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") @@ -231,10 +232,10 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): pure_llm_hallucin_sum = 0 gnn_save_list = [] untuned_llm_save_list = [] - for batch in tqdm(loader): + for i, batch in tqdm(loader): question = batch.question[0] correct_answer = batch.label[0] - gnn_llm_out = model.inference(batch) + gnn_llm_out = gnn_llm_eval_outs[i] # GNN+LLM only using 32 tokens to answer, give untrained LLM more pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) gnn_llm_pred = gnn_llm_out['pred'][0] @@ -276,7 +277,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" del model - if path.exists("llm.pt"): + if path.exists("llm.pt") and path.exists("llm_eval_outs.pt"): print("Existing finetuned LLAMA2 found.") print("Would you like to retrain?") user_input = str(input("(y/n):")).lower() @@ -286,13 +287,13 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): if retrain: print("Finetuning LLAMA2...") since = time.time() - _, _, pure_llm = train(since, 1, None, None, batch_size, + _, _, pure_llm_eval_outputs = train(since, 1, None, None, batch_size, eval_batch_size, lr, model=pure_llm, dataset=dataset) e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: - pure_llm = torch.load("llm.pt") + pure_llm_eval_outputs = torch.load("llm_eval_outs.pt") print("Evaluating Tuned LLM...") for i, batch in tqdm(enumerate(loader)): question = batch.question[0] @@ -301,7 +302,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm.inference(batch)['pred'][0] + pure_llm_pred = pure_llm_eval_outputs[i] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) if pure_llm_hallucinates == "skip": @@ -312,7 +313,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): final_prnt_str += "Label: " + correct_answer + "\n" final_prnt_str += "Untuned LLM Output: " + untuned_llm_pred + "\n" final_prnt_str += "Tuned LLM Output: " + pure_llm_pred + "\n" - final_prnt_str += "GNN+LLM Output:" + gnn_llm_pred + "\n" + final_prnt_str += "GNN+LLM Output: " + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" print("After finetuning the LLM...") print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) @@ -343,7 +344,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): args = parser.parse_args() # check if saved model retrain = True - if path.exists("gnn_llm.pt"): + if path.exists("gnn_llm.pt") and path.exists("gnn_llm_eval_outs.pt"): print("Existing trained model found.") print("Would you like to retrain?") user_input = str(input("(y/n):")).lower() @@ -352,7 +353,7 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): retrain = True if retrain: since = time.time() - prep_time, dataset, model = train(since, args.epochs, + prep_time, dataset, gnn_llm_eval_outs = train(since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, args.eval_batch_size, args.lr, @@ -364,9 +365,9 @@ def minimal_demo(model, dataset, lr, epochs, batch_size, eval_batch_size): print("E2E time (e2e_time) =", e2e_time, "seconds") print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") else: - model = torch.load("gnn_llm.pt") + gnn_llm_eval_outs = torch.load("gnn_llm_eval_outs.pt") dataset = WebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") print("First comparing against a pretrained LLAMA2 model") - minimal_demo(model, dataset, args.lr, args.epochs, args.batch_size, + minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) From 4c971f6d936926a55e4f151fe7dc72e9a70c4533 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:45:36 +0000 Subject: [PATCH 318/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 6d86d5ed3705..9d7c2b89cbf7 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -208,7 +208,8 @@ def adjust_learning_rate(param_group, LR, epoch): return prep_time, dataset, eval_output -def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size): +def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, + eval_batch_size): pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") @@ -288,8 +289,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ print("Finetuning LLAMA2...") since = time.time() _, _, pure_llm_eval_outputs = train(since, 1, None, None, batch_size, - eval_batch_size, lr, model=pure_llm, - dataset=dataset) + eval_batch_size, lr, + model=pure_llm, dataset=dataset) e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: @@ -353,11 +354,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ retrain = True if retrain: since = time.time() - prep_time, dataset, gnn_llm_eval_outs = train(since, args.epochs, - args.gnn_hidden_channels, - args.num_gnn_layers, args.batch_size, - args.eval_batch_size, args.lr, - checkpointing=args.checkpointing) + prep_time, dataset, gnn_llm_eval_outs = train( + since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, + args.batch_size, args.eval_batch_size, args.lr, + checkpointing=args.checkpointing) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -369,5 +369,5 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ dataset = WebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") print("First comparing against a pretrained LLAMA2 model") - minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, - args.eval_batch_size) + minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, + args.batch_size, args.eval_batch_size) From 9356bd9fc778962122db2c52b4e7449b94fafee9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 09:56:08 -0700 Subject: [PATCH 319/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 6d86d5ed3705..678a07d91827 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -225,20 +225,22 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ # batch size 1 loader for simplicity loader = DataLoader(test_dataset, batch_size=1, drop_last=False, pin_memory=True, shuffle=False) - # Step loop through the loader and run both models - print( - "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") gnn_llm_hallucin_sum = 0 pure_llm_hallucin_sum = 0 gnn_save_list = [] untuned_llm_save_list = [] + print("Extracting necesarry info from saved GNN+LLM outputs") + gnn_llm_preds = [] + for out in tqdm(gnn_llm_eval_outs): + gnn_llm_preds += out['pred'] + print( + "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") for i, batch in tqdm(loader): question = batch.question[0] correct_answer = batch.label[0] - gnn_llm_out = gnn_llm_eval_outs[i] # GNN+LLM only using 32 tokens to answer, give untrained LLM more pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) - gnn_llm_pred = gnn_llm_out['pred'][0] + gnn_llm_pred = gnn_llm_preds[i] pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) @@ -294,7 +296,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm_eval_outputs = torch.load("llm_eval_outs.pt") - print("Evaluating Tuned LLM...") + print("Extracting necesarry info from finetuned LLM outputs...") + pure_llm_preds = [] + for out in tqdm(pure_llm_eval_outs): + pure_llm_preds += out['pred'] for i, batch in tqdm(enumerate(loader)): question = batch.question[0] correct_answer = batch.label[0] @@ -302,7 +307,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_ untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm_eval_outputs[i] + pure_llm_pred = pure_llm_preds[i] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) if pure_llm_hallucinates == "skip": From 54e76911fb0b81fff068161f952af83fdbd4b5cb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:00:22 -0700 Subject: [PATCH 320/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index e8c4f58f5f09..fd3571ed7701 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -210,6 +210,7 @@ def adjust_learning_rate(param_group, LR, epoch): def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size): + print("First comparing against a pretrained LLAMA2 model...") pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") @@ -258,10 +259,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) percent = 100.0 * round( 1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) - print(f"GNN reduces hallucinations by: ~{percent}%") + print(f"GNN reduces pretrained LLM hallucinations by: ~{percent}%") print("Note: hallucinations detected by regex hence the ~") print("Now we see how the LLM compares when finetuned...") - print("Saving outputs of GNN+LLM and Pretrained LLM...") + print("Saving outputs of GNN+LLM and pretrained LLM...") save_dict = { "gnn_save_list": gnn_save_list, "untuned_llm_save_list": untuned_llm_save_list, @@ -269,6 +270,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "pure_llm_hallucin_sum": pure_llm_hallucin_sum } torch.save(save_dict, "demo_save_dict.pt") + print("Done!") else: save_dict = torch.load("demo_save_dict.pt") gnn_save_list = save_dict["gnn_save_list"] @@ -301,6 +303,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_preds = [] for out in tqdm(pure_llm_eval_outs): pure_llm_preds += out['pred'] + print("Final comparison between all models...") for i, batch in tqdm(enumerate(loader)): question = batch.question[0] correct_answer = batch.label[0] @@ -321,7 +324,6 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, final_prnt_str += "Tuned LLM Output: " + pure_llm_pred + "\n" final_prnt_str += "GNN+LLM Output: " + gnn_llm_pred + "\n" final_prnt_str += "#" * 20 + "\n" - print("After finetuning the LLM...") print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) @@ -373,6 +375,5 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, gnn_llm_eval_outs = torch.load("gnn_llm_eval_outs.pt") dataset = WebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") - print("First comparing against a pretrained LLAMA2 model") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size) From c4b6a8faf809a7656bc152cc8ed7d96cbe6dde2d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:04:15 -0700 Subject: [PATCH 321/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index fd3571ed7701..843def64830f 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -231,7 +231,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_hallucin_sum = 0 gnn_save_list = [] untuned_llm_save_list = [] - print("Extracting necesarry info from saved GNN+LLM outputs") + print("Extracting necesarry info from saved GNN+LLM outputs...") gnn_llm_preds = [] for out in tqdm(gnn_llm_eval_outs): gnn_llm_preds += out['pred'] From c37a52e7298cf3f711c2a822bd97ba2b75511fbd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:12:05 -0700 Subject: [PATCH 322/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 843def64830f..e4332c18fd2a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -237,7 +237,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, gnn_llm_preds += out['pred'] print( "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") - for i, batch in tqdm(loader): + for i, batch in tqdm(enumerate(loader)): question = batch.question[0] correct_answer = batch.label[0] # GNN+LLM only using 32 tokens to answer, give untrained LLM more @@ -281,7 +281,6 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, trained_llm_hallucin_sum = 0 untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" - del model if path.exists("llm.pt") and path.exists("llm_eval_outs.pt"): print("Existing finetuned LLAMA2 found.") print("Would you like to retrain?") From bcdc98893e753ccef6b1be30488955b2f8ac6007 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:18:04 -0700 Subject: [PATCH 323/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index e4332c18fd2a..536a52c38045 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -237,7 +237,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, gnn_llm_preds += out['pred'] print( "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") - for i, batch in tqdm(enumerate(loader)): + for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] # GNN+LLM only using 32 tokens to answer, give untrained LLM more @@ -303,7 +303,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, for out in tqdm(pure_llm_eval_outs): pure_llm_preds += out['pred'] print("Final comparison between all models...") - for i, batch in tqdm(enumerate(loader)): + for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] From 46aca67a2c84880bbf2ee71e92fd717c65498b2c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:19:16 -0700 Subject: [PATCH 324/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 536a52c38045..74f7e5d71699 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -231,9 +231,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_hallucin_sum = 0 gnn_save_list = [] untuned_llm_save_list = [] - print("Extracting necesarry info from saved GNN+LLM outputs...") gnn_llm_preds = [] - for out in tqdm(gnn_llm_eval_outs): + for out in gnn_llm_eval_outs: gnn_llm_preds += out['pred'] print( "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") @@ -298,9 +297,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("E2E time (e2e_time) =", e2e_time, "seconds") else: pure_llm_eval_outputs = torch.load("llm_eval_outs.pt") - print("Extracting necesarry info from finetuned LLM outputs...") pure_llm_preds = [] - for out in tqdm(pure_llm_eval_outs): + for out in pure_llm_eval_outs: pure_llm_preds += out['pred'] print("Final comparison between all models...") for i, batch in enumerate(tqdm(loader)): From b80467f38aac9af7e5531f325447eaea7a0d888d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:21:27 -0700 Subject: [PATCH 325/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 74f7e5d71699..f698cc87259d 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -19,7 +19,7 @@ from tqdm import tqdm from torch_geometric import seed_everything -from torch_geometric.data import DataLoader +from torch_geometric.loader import DataLoader from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM From b24a8a498858d940527f38ad7099b9c17b1c1579 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:23:23 +0000 Subject: [PATCH 326/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f698cc87259d..7e41e7281553 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -19,8 +19,8 @@ from tqdm import tqdm from torch_geometric import seed_everything -from torch_geometric.loader import DataLoader from torch_geometric.datasets import WebQSPDataset +from torch_geometric.loader import DataLoader from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM From c9a2a391cb75f1f7b9bb11928c4ded9b0753a188 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:44:19 -0700 Subject: [PATCH 327/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f698cc87259d..fbaedd5629ff 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -298,7 +298,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, else: pure_llm_eval_outputs = torch.load("llm_eval_outs.pt") pure_llm_preds = [] - for out in pure_llm_eval_outs: + for out in pure_llm_eval_outputs: pure_llm_preds += out['pred'] print("Final comparison between all models...") for i, batch in enumerate(tqdm(loader)): From 6b8fa8f7d9d60d36ff042c327bdc361db37092dd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 10:46:12 -0700 Subject: [PATCH 328/752] cleanup --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2284321c2284..0fb69fa4729c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -33,13 +33,13 @@ from torch_geometric.data import Data, InMemoryDataset +# Alot of code in this file is based on the original G-Retriever paper +# url: https://arxiv.org/abs/2402.07630 @no_type_check def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: - # from original G-Retriever work - # https://arxiv.org/abs/2402.07630 c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( From 244c032eb0773d38fc74a6d1bda9a4c135b24033 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:47:41 +0000 Subject: [PATCH 329/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0fb69fa4729c..fb83ff7232c1 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -36,6 +36,7 @@ # Alot of code in this file is based on the original G-Retriever paper # url: https://arxiv.org/abs/2402.07630 + @no_type_check def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, textual_edges: df, topk: int = 3, topk_e: int = 3, From 3bf8744ce1c65fa96757cd38be9a9fa99e34040d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 12:32:26 -0700 Subject: [PATCH 330/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 6a426266ec9d..3116e1efe6d6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -211,6 +211,12 @@ def adjust_learning_rate(param_group, LR, epoch): def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size): print("First comparing against a pretrained LLAMA2 model...") + # Step 1: Define a single batch size test loader + idx_split = dataset.split_idxs + test_dataset = [dataset[i] for i in idx_split['test']] + # batch size 1 loader for simplicity + loader = DataLoader(test_dataset, batch_size=1, drop_last=False, + pin_memory=True, shuffle=False) pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") @@ -221,12 +227,6 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, skip_step_one = False if not skip_step_one: - # Step 1: Define a single batch size test loader - idx_split = dataset.split_idxs - test_dataset = [dataset[i] for i in idx_split['test']] - # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=1, drop_last=False, - pin_memory=True, shuffle=False) gnn_llm_hallucin_sum = 0 pure_llm_hallucin_sum = 0 gnn_save_list = [] From 3014f7247dad68b1a5ad1e793f64424414983c0c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 12:35:10 -0700 Subject: [PATCH 331/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3116e1efe6d6..14ae59343995 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -324,6 +324,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) + percent = 100.0 * round( + 1 - (gnn_llm_hallucin_sum / untuned_llm_hallucin_sum), 2) print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") tuned_percent = 100.0 * round( 1 - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) From b511049b31d6b1bf251c307abf66543f67c0a032 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:36:29 +0000 Subject: [PATCH 332/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 14ae59343995..ec209861c67b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -325,7 +325,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) percent = 100.0 * round( - 1 - (gnn_llm_hallucin_sum / untuned_llm_hallucin_sum), 2) + 1 - (gnn_llm_hallucin_sum / untuned_llm_hallucin_sum), 2) print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") tuned_percent = 100.0 * round( 1 - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) From b81819e5427ef280d5de4a9fdbec29a58668b95d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 12:38:34 -0700 Subject: [PATCH 333/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 14ae59343995..94bab2f669ec 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -331,7 +331,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, 1 - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) print(f"GNN reduces tuned LLM hallucinations by: ~{tuned_percent}%") print("Note: hallucinations detected by regex hence the ~") - print("Potential instances where GNN solves the hallucinations of LLM") + print("Potential instances where GNN solves the hallucinations of LLM:") print(final_prnt_str) From 5bff5d70fac8483faf0641c38809112e3b5be495 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:04:49 -0700 Subject: [PATCH 334/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index c34f8f1ede6a..834c1e226bd2 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -315,12 +315,12 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, continue trained_llm_hallucin_sum += int(pure_llm_hallucinates) if pure_llm_hallucinates and not gnn_llm_hallucinates: - final_prnt_str += "Prompt: " + question + "\n" - final_prnt_str += "Label: " + correct_answer + "\n" - final_prnt_str += "Untuned LLM Output: " + untuned_llm_pred + "\n" - final_prnt_str += "Tuned LLM Output: " + pure_llm_pred + "\n" - final_prnt_str += "GNN+LLM Output: " + gnn_llm_pred + "\n" - final_prnt_str += "#" * 20 + "\n" + final_prnt_str += "Prompt: '" + question + "'\n" + final_prnt_str += "Label: '" + correct_answer + "'\n" + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" + final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" + final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "\n'" + final_prnt_str += "#" * 20 + "\n\n" print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) From 69bd39cf9eb9d6b8d141041a70d3f592a9bbeba8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:06:54 -0700 Subject: [PATCH 335/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 834c1e226bd2..69afcb592233 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -320,7 +320,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "\n'" - final_prnt_str += "#" * 20 + "\n\n" + final_prnt_str += "\n#" * 20 + "\n\n" print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) From f1769e81766c29aa4d63fa9901c7953dfd5e98dd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:08:16 -0700 Subject: [PATCH 336/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 69afcb592233..4b8bd1357edc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -320,7 +320,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "\n'" - final_prnt_str += "\n#" * 20 + "\n\n" + final_prnt_str += "\n" + "#" * 20 + "\n\n" print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) From d2486a33accbce7cdfae8058293a3b985ecdf13a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:09:44 -0700 Subject: [PATCH 337/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 4b8bd1357edc..c3472349f12a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -319,7 +319,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, final_prnt_str += "Label: '" + correct_answer + "'\n" final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" - final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "\n'" + final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" final_prnt_str += "\n" + "#" * 20 + "\n\n" print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) From 875d93cd5ddbe7d2d40d25463c387b8ea7d5f196 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:27:20 -0700 Subject: [PATCH 338/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index c3472349f12a..5dc7655ca640 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -314,7 +314,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if pure_llm_hallucinates == "skip": continue trained_llm_hallucin_sum += int(pure_llm_hallucinates) - if pure_llm_hallucinates and not gnn_llm_hallucinates: + if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa final_prnt_str += "Prompt: '" + question + "'\n" final_prnt_str += "Label: '" + correct_answer + "'\n" final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" From 06899d6fe7e4dbdad9f27f666b6a04e7afac9c4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 20:28:24 +0000 Subject: [PATCH 339/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5dc7655ca640..15bb0e792b61 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -314,7 +314,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if pure_llm_hallucinates == "skip": continue trained_llm_hallucin_sum += int(pure_llm_hallucinates) - if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa + if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa final_prnt_str += "Prompt: '" + question + "'\n" final_prnt_str += "Label: '" + correct_answer + "'\n" final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" From 0ce56eebcb430a4211258907e0e516c5a4a7b207 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 13:38:26 -0700 Subject: [PATCH 340/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5dc7655ca640..2823b3afd822 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -317,7 +317,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa final_prnt_str += "Prompt: '" + question + "'\n" final_prnt_str += "Label: '" + correct_answer + "'\n" - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" final_prnt_str += "\n" + "#" * 20 + "\n\n" From 7d4346477be20d11e520ba9f5ccf6a424623e97d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 20:39:52 +0000 Subject: [PATCH 341/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ae9c24afeea6..12b28a306fe5 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -317,7 +317,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa final_prnt_str += "Prompt: '" + question + "'\n" final_prnt_str += "Label: '" + correct_answer + "'\n" - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" final_prnt_str += "\n" + "#" * 20 + "\n\n" From edb0fbc78a9d923d81547b887b19895532da3d00 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 14:51:18 -0700 Subject: [PATCH 342/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 12b28a306fe5..d269de76eafc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -179,14 +179,15 @@ def adjust_learning_rate(param_group, LR, epoch): if checkpointing and val_loss < best_val_loss: print("Checkpointing best val loss model...") best_val_loss = val_loss - torch.save(model, model_save_name + "_best_val_loss_ckpt.pt") + torch.save(model.state_dict(), model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() # Step 5 Evaluating print("Final Evaluation...") if checkpointing: - model = torch.load(model_save_name + "_best_val_loss_ckpt.pt") + state_dict = torch.load(model_save_name + "_best_val_loss_ckpt.pt") + model = model.load_state_dict(state_dict) model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) @@ -201,7 +202,7 @@ def adjust_learning_rate(param_group, LR, epoch): print(f'Test Acc {acc}') # save model print("Saving Model...") - torch.save(model, model_save_name + ".pt") + torch.save(model.state_dict(), model_save_name + ".pt") print("Saving eval output for downstream demo...") torch.save(eval_output, model_save_name + "_eval_outs.pt") print("Done!") From 574418ece7d1d352d1d4219510028da71d8ecd84 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 21:52:23 +0000 Subject: [PATCH 343/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d269de76eafc..5ba375e14a79 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -179,7 +179,8 @@ def adjust_learning_rate(param_group, LR, epoch): if checkpointing and val_loss < best_val_loss: print("Checkpointing best val loss model...") best_val_loss = val_loss - torch.save(model.state_dict(), model_save_name + "_best_val_loss_ckpt.pt") + torch.save(model.state_dict(), + model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() From ef0ef472bb22269c7be6b98f2e56576760412230 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 15:24:41 -0700 Subject: [PATCH 344/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d269de76eafc..8027693e3a49 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -139,6 +139,7 @@ def adjust_learning_rate(param_group, LR, epoch): best_val_loss = float('inf') # Step 4 Training + best_epoch = 0 for epoch in range(num_epochs): model.train() epoch_loss = 0. @@ -179,13 +180,14 @@ def adjust_learning_rate(param_group, LR, epoch): if checkpointing and val_loss < best_val_loss: print("Checkpointing best val loss model...") best_val_loss = val_loss + best_epoch = epoch torch.save(model.state_dict(), model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() # Step 5 Evaluating print("Final Evaluation...") - if checkpointing: + if checkpointing and best_epoch != num_epochs - 1: state_dict = torch.load(model_save_name + "_best_val_loss_ckpt.pt") model = model.load_state_dict(state_dict) model.eval() From ff6e52dced9002d0598dd3fef6ac4eccee95462a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 15:31:00 -0700 Subject: [PATCH 345/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 975b896bf3ec..ed8dab0c8ed0 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -187,13 +187,14 @@ def adjust_learning_rate(param_group, LR, epoch): torch.cuda.reset_max_memory_allocated() # Step 5 Evaluating - print("Final Evaluation...") if checkpointing and best_epoch != num_epochs - 1: + print("Loading best checkpoint...") state_dict = torch.load(model_save_name + "_best_val_loss_ckpt.pt") model = model.load_state_dict(state_dict) model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) + print("Final Evaluation...") for step, batch in enumerate(test_loader): with torch.no_grad(): output = model.inference(batch) From 740ab5e3a01a222f6d4daeee812d5ad0cc373052 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 15:38:17 -0700 Subject: [PATCH 346/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ed8dab0c8ed0..10ddb1f67164 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -80,6 +80,21 @@ def compute_accuracy(eval_output): return hit +def save_params_dict(model, save_path): + state_dict = model.state_dict() + for k in list(state_dict.keys()): + if k in param_grad_dic.keys() and not param_grad_dic[k]: + # delete parameters that do not require gradient + del state_dict[k] + torch.save(state_dict, save_path) + + +def load_params_dict(model, savepath): + state_dict = torch.load(save_path) + model.load_state_dict(state_dict) + return model + + def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, model=None, dataset=None, checkpointing=False): def adjust_learning_rate(param_group, LR, epoch): @@ -181,7 +196,7 @@ def adjust_learning_rate(param_group, LR, epoch): print("Checkpointing best val loss model...") best_val_loss = val_loss best_epoch = epoch - torch.save(model.state_dict(), + save_params_dict(model, model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() @@ -189,8 +204,8 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 5 Evaluating if checkpointing and best_epoch != num_epochs - 1: print("Loading best checkpoint...") - state_dict = torch.load(model_save_name + "_best_val_loss_ckpt.pt") - model = model.load_state_dict(state_dict) + model = load_params_dict(model, + model_save_name + "_best_val_loss_ckpt.pt") model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) @@ -206,7 +221,7 @@ def adjust_learning_rate(param_group, LR, epoch): print(f'Test Acc {acc}') # save model print("Saving Model...") - torch.save(model.state_dict(), model_save_name + ".pt") + save_params_dict(model, model_save_name + ".pt") print("Saving eval output for downstream demo...") torch.save(eval_output, model_save_name + "_eval_outs.pt") print("Done!") From 0bdd98c9fc379a0d1083044bd15baa49b9e6dc79 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 22:39:29 +0000 Subject: [PATCH 347/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 10ddb1f67164..edc6409d0041 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -196,8 +196,7 @@ def adjust_learning_rate(param_group, LR, epoch): print("Checkpointing best val loss model...") best_val_loss = val_loss best_epoch = epoch - save_params_dict(model, - model_save_name + "_best_val_loss_ckpt.pt") + save_params_dict(model, model_save_name + "_best_val_loss_ckpt.pt") torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() @@ -205,7 +204,7 @@ def adjust_learning_rate(param_group, LR, epoch): if checkpointing and best_epoch != num_epochs - 1: print("Loading best checkpoint...") model = load_params_dict(model, - model_save_name + "_best_val_loss_ckpt.pt") + model_save_name + "_best_val_loss_ckpt.pt") model.eval() eval_output = [] progress_bar_test = tqdm(range(len(test_loader))) From b6d194858b897e14949fad18d87d65e41a694641 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 15:50:54 -0700 Subject: [PATCH 348/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 10ddb1f67164..f9c9cce214bb 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -82,8 +82,11 @@ def compute_accuracy(eval_output): def save_params_dict(model, save_path): state_dict = model.state_dict() + param_grad_dict = { + k: v.requires_grad for (k, v) in model.named_parameters() + } for k in list(state_dict.keys()): - if k in param_grad_dic.keys() and not param_grad_dic[k]: + if k in param_grad_dict.keys() and not param_grad_dict[k]: # delete parameters that do not require gradient del state_dict[k] torch.save(state_dict, save_path) From 4d37e0c9cfc124c32827dff966951c688cc4e736 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 22:52:16 +0000 Subject: [PATCH 349/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5b11d6a7bed1..5ec0fd296131 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -83,7 +83,8 @@ def compute_accuracy(eval_output): def save_params_dict(model, save_path): state_dict = model.state_dict() param_grad_dict = { - k: v.requires_grad for (k, v) in model.named_parameters() + k: v.requires_grad + for (k, v) in model.named_parameters() } for k in list(state_dict.keys()): if k in param_grad_dict.keys() and not param_grad_dict[k]: From dac44f0b31283ff5015bfe2dfa5732e2492e14b6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 16 Apr 2024 16:14:06 -0700 Subject: [PATCH 350/752] cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 5b11d6a7bed1..1f61e56b15e0 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -92,7 +92,7 @@ def save_params_dict(model, save_path): torch.save(state_dict, save_path) -def load_params_dict(model, savepath): +def load_params_dict(model, save_path): state_dict = torch.load(save_path) model.load_state_dict(state_dict) return model From 4f34991f0e8b8892cfe4f49557801e5e37658a7a Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 09:05:32 -0700 Subject: [PATCH 351/752] addressing reviews --- torch_geometric/nn/models/gnn_llm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a66a4a57aae1..a34635080a81 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -73,6 +73,13 @@ def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, else: self.printable_llm_name = llm_name self.huggingface_str = llm_name + + """ + This is a rough hueristic: + We found that LLAMA2 (7B) + GAT hits OOM + on a single 80GB GPU, but can fit on a single + GPU that is slightly larger GPU. + """ self.mem_needed = 85 * num_params / 7 self.llm_dtype = llm_dtype print('Loading ' + str(self.printable_llm_name)) From b875e2bbeb7499eb394d2c93dbdb6f3796f04d4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:06:33 +0000 Subject: [PATCH 352/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index a34635080a81..3c18fc417ec5 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -73,7 +73,6 @@ def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, else: self.printable_llm_name = llm_name self.huggingface_str = llm_name - """ This is a rough hueristic: We found that LLAMA2 (7B) + GAT hits OOM From d5b81a5725ceb194c292b1844085f39506bbcabb Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 11:10:50 -0700 Subject: [PATCH 353/752] addressing suggested change for LLM param names --- torch_geometric/nn/models/gnn_llm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 3c18fc417ec5..9f69a35c96fc 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -61,13 +61,13 @@ def get_llm_kwargs(mem_needed): class LLM(nn.Module): - def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, + def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, num_params: int = 7): super().__init__() - if llm_name == "llama2": + if model_name == "llama2": self.printable_llm_name = "LLAMA2" self.huggingface_str = llama2_str_name - elif llm_name == "gemma": + elif model_name == "gemma": self.printable_llm_name = "GEMMA" self.huggingface_str = gemma_str_name else: @@ -80,7 +80,7 @@ def __init__(self, llm_name: str = "llama2", llm_dtype=torch.bfloat16, GPU that is slightly larger GPU. """ self.mem_needed = 85 * num_params / 7 - self.llm_dtype = llm_dtype + self.llm_dtype = dtype print('Loading ' + str(self.printable_llm_name)) kwargs = get_llm_kwargs(self.mem_needed) print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) From 1adacb2df0661f714b42784a27443acfea5b626d Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 11:11:40 -0700 Subject: [PATCH 354/752] commiting suggestion Co-authored-by: Matthias Fey --- torch_geometric/nn/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index a89c9c9a9539..f792cf15f14a 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -42,5 +42,5 @@ 'MetaPath2Vec', 'DeepGCNLayer', 'TGNMemory', 'LabelPropagation', 'CorrectAndSmooth', 'AttentiveFP', 'RECT_L', 'LINKX', 'LightGCN', 'MaskLabel', 'GroupAddRev', 'GNNFF', 'PMLP', 'NeuralFingerprint', 'ViSNet', - 'GNN_LLM' + 'GNN_LLM', ] From 89bd46ce99e729e2f4261f4b2a23e3fc8b941e83 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 11:12:49 -0700 Subject: [PATCH 355/752] commiting suggestion to fix typo Co-authored-by: Matthias Fey --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index fb83ff7232c1..2f015280cabd 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -165,7 +165,7 @@ def __getitem__(self, index: int) -> Dict[str, torch.Tensor]: class Sentence_Transformer(torch.nn.Module): def __init__(self, pretrained_repo: str) -> None: - super(Sentence_Transformer, self).__init__() + super().__init__() print(f"inherit model weights from {pretrained_repo}") self.bert_model = AutoModel.from_pretrained(pretrained_repo) From db82d71072acd509623d79c007dbfc13afeafd67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:12:52 +0000 Subject: [PATCH 356/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/__init__.py | 50 ++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index f792cf15f14a..b9f315fd192e 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -34,13 +34,45 @@ captum_output_to_dicts) __all__ = classes = [ - 'MLP', 'GCN', 'GraphSAGE', 'GIN', 'GAT', 'PNA', 'EdgeCNN', - 'JumpingKnowledge', 'MetaLayer', 'Node2Vec', 'DeepGraphInfomax', - 'InnerProductDecoder', 'GAE', 'VGAE', 'ARGA', 'ARGVA', 'SignedGCN', - 'RENet', 'GraphUNet', 'SchNet', 'DimeNet', 'DimeNetPlusPlus', - 'to_captum_model', 'to_captum_input', 'captum_output_to_dicts', - 'MetaPath2Vec', 'DeepGCNLayer', 'TGNMemory', 'LabelPropagation', - 'CorrectAndSmooth', 'AttentiveFP', 'RECT_L', 'LINKX', 'LightGCN', - 'MaskLabel', 'GroupAddRev', 'GNNFF', 'PMLP', 'NeuralFingerprint', 'ViSNet', - 'GNN_LLM', + 'MLP', + 'GCN', + 'GraphSAGE', + 'GIN', + 'GAT', + 'PNA', + 'EdgeCNN', + 'JumpingKnowledge', + 'MetaLayer', + 'Node2Vec', + 'DeepGraphInfomax', + 'InnerProductDecoder', + 'GAE', + 'VGAE', + 'ARGA', + 'ARGVA', + 'SignedGCN', + 'RENet', + 'GraphUNet', + 'SchNet', + 'DimeNet', + 'DimeNetPlusPlus', + 'to_captum_model', + 'to_captum_input', + 'captum_output_to_dicts', + 'MetaPath2Vec', + 'DeepGCNLayer', + 'TGNMemory', + 'LabelPropagation', + 'CorrectAndSmooth', + 'AttentiveFP', + 'RECT_L', + 'LINKX', + 'LightGCN', + 'MaskLabel', + 'GroupAddRev', + 'GNNFF', + 'PMLP', + 'NeuralFingerprint', + 'ViSNet', + 'GNN_LLM', ] From 1f4c9490d38070f7c77ae5109253711af8910e21 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 11:27:51 -0700 Subject: [PATCH 357/752] cleanup sentence_transformer --- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2f015280cabd..aedd4a8b80a9 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -163,7 +163,7 @@ def __getitem__(self, index: int) -> Dict[str, torch.Tensor]: return batch_data -class Sentence_Transformer(torch.nn.Module): +class SentenceTransformer(torch.nn.Module): def __init__(self, pretrained_repo: str) -> None: super().__init__() print(f"inherit model weights from {pretrained_repo}") @@ -189,7 +189,7 @@ def forward(self, input_ids: torch.Tensor, return sentence_embeddings -def sbert_text2embedding(model: Sentence_Transformer, +def sbert_text2embedding(model: SentenceTransformer, tokenizer: torch.nn.Module, device: torch.device, text: List[str]) -> torch.Tensor: try: @@ -300,7 +300,7 @@ def download(self) -> None: def process(self) -> None: pretrained_repo = "sentence-transformers/all-roberta-large-v1" - self.model = Sentence_Transformer(pretrained_repo) + self.model = SentenceTransformer(pretrained_repo) self.model.to(self.device) self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) From 59fae5e7008614974ea79a20b1f495c9873d20a8 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 24 Apr 2024 11:54:46 -0700 Subject: [PATCH 358/752] fixing type hint --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index aedd4a8b80a9..324a471d7d42 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Tuple, no_type_check +from typing import Dict, List, Tuple, no_type_check, Union import numpy as np @@ -153,7 +153,7 @@ def __init__(self, input_ids: torch.Tensor, def __len__(self) -> int: return self.data["input_ids"].size(0) - def __getitem__(self, index: int) -> Dict[str, torch.Tensor]: + def __getitem__(self, index: Union[int, torch.Tensor]) -> Dict[str, torch.Tensor]: if isinstance(index, torch.Tensor): index = index.item() batch_data = dict() From 84ced021367ce5cfe001193878c0db3922c81ab3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:55:51 +0000 Subject: [PATCH 359/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 324a471d7d42..92ca2bf0ef87 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Tuple, no_type_check, Union +from typing import Dict, List, Tuple, Union, no_type_check import numpy as np @@ -153,7 +153,8 @@ def __init__(self, input_ids: torch.Tensor, def __len__(self) -> int: return self.data["input_ids"].size(0) - def __getitem__(self, index: Union[int, torch.Tensor]) -> Dict[str, torch.Tensor]: + def __getitem__( + self, index: Union[int, torch.Tensor]) -> Dict[str, torch.Tensor]: if isinstance(index, torch.Tensor): index = index.item() batch_data = dict() From 1a74985189561a6b74bff45d66ee7376e9b3308c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 24 Apr 2024 11:58:39 -0700 Subject: [PATCH 360/752] rename GNN_LLM to GRetriever --- examples/llm_plus_gnn/g_retriever.py | 2 +- torch_geometric/nn/models/gnn_llm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 612e51016438..027464b3bb39 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -133,7 +133,7 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: - model = GNN_LLM(gnn_hidden_channels=hidden_channels, + model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) if num_gnn_layers is not None: model_save_name = "gnn_llm" diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 9f69a35c96fc..f3c45a9397fe 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -216,7 +216,7 @@ def inference(self, samples: Batch, max_out_tokens=max_new_tokens): } -class GNN_LLM(nn.Module): +class GRetriever(nn.Module): r"""This GNN+LLM implementation is based on G-retriever. Original Paper: `_. See `examples/llm_plus_gnn/g_retriever.py` for example usage. @@ -266,7 +266,7 @@ def __init__( super().__init__() if not WITH_TRANSFORMERS: raise ImportError( - "To use GNN_LLM, please `pip install transformers`.") + "To use GRetriever, please `pip install transformers`.") if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) elif 'gemma' in llm_to_use.lower(): From 04ca2f8666ddad940fb60986c06eadabffd337f0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 19:00:30 +0000 Subject: [PATCH 361/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 027464b3bb39..86f23075a6b8 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -134,7 +134,7 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: model = GRetriever(gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers) + num_gnn_layers=num_gnn_layers) if num_gnn_layers is not None: model_save_name = "gnn_llm" else: From 3c755e4b7605b2ae5ab3686a80e250ac99c43315 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 24 Apr 2024 13:50:05 -0700 Subject: [PATCH 362/752] cleaning imports --- torch_geometric/datasets/web_qsp_dataset.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 92ca2bf0ef87..f5b85ad06631 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -20,17 +20,6 @@ from torch.utils.data import DataLoader from tqdm import tqdm -try: - from transformers import AutoModel, AutoTokenizer - WITH_TRANSFORMERS = True -except ImportError as e: # noqa - WITH_TRANSFORMERS = False -try: - import datasets - WITH_DATASETS = True -except ImportError as e: # noqa - WITH_DATASETS = False - from torch_geometric.data import Data, InMemoryDataset # Alot of code in this file is based on the original G-Retriever paper @@ -252,15 +241,11 @@ def __init__( ) -> None: missing_imports = False missing_str_list = [] + from transformers import AutoModel, AutoTokenizer + import datasets if not WITH_PCST: missing_str_list.append('pcst_fast') missing_imports = True - if not WITH_TRANSFORMERS: - missing_str_list.append('transformers') - missing_imports = True - if not WITH_DATASETS: - missing_str_list.append('datasets') - missing_imports = True if not WITH_PANDAS: missing_str_list.append('pandas') missing_imports = True From a99a1b95c2ba00bea2e3762986e32491564c2f83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 20:51:17 +0000 Subject: [PATCH 363/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f5b85ad06631..c90886922369 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -241,8 +241,8 @@ def __init__( ) -> None: missing_imports = False missing_str_list = [] - from transformers import AutoModel, AutoTokenizer import datasets + from transformers import AutoModel, AutoTokenizer if not WITH_PCST: missing_str_list.append('pcst_fast') missing_imports = True From 1bde6bfff2ff4a51c22acad8fd8f711a4fe9418c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 24 Apr 2024 13:51:22 -0700 Subject: [PATCH 364/752] reverting --- torch_geometric/datasets/web_qsp_dataset.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f5b85ad06631..92ca2bf0ef87 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -20,6 +20,17 @@ from torch.utils.data import DataLoader from tqdm import tqdm +try: + from transformers import AutoModel, AutoTokenizer + WITH_TRANSFORMERS = True +except ImportError as e: # noqa + WITH_TRANSFORMERS = False +try: + import datasets + WITH_DATASETS = True +except ImportError as e: # noqa + WITH_DATASETS = False + from torch_geometric.data import Data, InMemoryDataset # Alot of code in this file is based on the original G-Retriever paper @@ -241,11 +252,15 @@ def __init__( ) -> None: missing_imports = False missing_str_list = [] - from transformers import AutoModel, AutoTokenizer - import datasets if not WITH_PCST: missing_str_list.append('pcst_fast') missing_imports = True + if not WITH_TRANSFORMERS: + missing_str_list.append('transformers') + missing_imports = True + if not WITH_DATASETS: + missing_str_list.append('datasets') + missing_imports = True if not WITH_PANDAS: missing_str_list.append('pandas') missing_imports = True From 3bbe694d58e8e0222acaee4e9e911931aea2ec45 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 24 Apr 2024 13:55:37 -0700 Subject: [PATCH 365/752] reverting --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 92ca2bf0ef87..0e52a1538575 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -3,7 +3,6 @@ import numpy as np try: - import pandas as pd from pandas import DataFrame as df WITH_PANDAS = True except ImportError as e: # noqa @@ -300,6 +299,7 @@ def download(self) -> None: } def process(self) -> None: + import pandas pretrained_repo = "sentence-transformers/all-roberta-large-v1" self.model = SentenceTransformer(pretrained_repo) self.model.to(self.device) From e3dc5dd024355c22574bf946195708001bfea75f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 24 Apr 2024 16:14:35 -0700 Subject: [PATCH 366/752] docstring for LLM --- torch_geometric/nn/models/gnn_llm.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index f3c45a9397fe..65d98eeb930e 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -61,6 +61,24 @@ def get_llm_kwargs(mem_needed): class LLM(nn.Module): + """ + This module wraps a HuggingFace Transformer based model in such + a way that makes it easy to use with PyG GNNs. + model_name (str): A string representing the huggingface model you + want to use. This module has been tested for 'llama2' and 'gemma'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on + https://github.com/pyg-team/pytorch_geometric + and assign to puririshi98. (default: :obj:'llama2') + dtype (torch.dtype): The dtype to use for the LLM. + (default :obj: `torch.bloat16`) + num_params (int): An integer representing how many params your + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the + available GPU memory of your GPUs (default :obj:`7`) + + """ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, num_params: int = 7): super().__init__() @@ -231,8 +249,8 @@ class GRetriever(nn.Module): and assign to puririshi98. (default: :obj:'llama2') llm_use_lora (bool): use LORA from peft for training the LLM. see https://huggingface.co/docs/peft/en/index for details. - llm_dtype (torch.dtype): The lower precision dtype to use for the - LLM. (default :obj: `torch.bloat16`) + llm_dtype (torch.dtype): The dtype to use for the LLM. + (default :obj: `torch.bloat16`) num_llm_params (int): An integer representing how many params your huggingface transformer model has, in billions. This is used to automatically allocate the number of gpus needed, given the From 8b327d099a1972ead1ba7de27d8d74871a510eb2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 23:16:24 +0000 Subject: [PATCH 367/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/gnn_llm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py index 65d98eeb930e..703d1318629f 100644 --- a/torch_geometric/nn/models/gnn_llm.py +++ b/torch_geometric/nn/models/gnn_llm.py @@ -61,8 +61,7 @@ def get_llm_kwargs(mem_needed): class LLM(nn.Module): - """ - This module wraps a HuggingFace Transformer based model in such + """This module wraps a HuggingFace Transformer based model in such a way that makes it easy to use with PyG GNNs. model_name (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. From 2463a43273c7c329a0b4b2c9757028f29f1a9825 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Sat, 27 Apr 2024 09:08:13 -0700 Subject: [PATCH 368/752] Cleanup (#9242) cleaning up to address review for https://github.com/pyg-team/pytorch_geometric/pull/9167/ --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- examples/llm_plus_gnn/g_retriever.py | 30 +- torch_geometric/datasets/web_qsp_dataset.py | 119 +---- torch_geometric/nn/models/__init__.py | 4 +- torch_geometric/nn/models/g_retriever.py | 310 ++++++++++++ torch_geometric/nn/models/gnn_llm.py | 472 ------------------ torch_geometric/nn/text/__init__.py | 10 + torch_geometric/nn/text/llm.py | 243 +++++++++ .../nn/text/sentence_transformer_embedding.py | 71 +++ 8 files changed, 670 insertions(+), 589 deletions(-) create mode 100644 torch_geometric/nn/models/g_retriever.py delete mode 100644 torch_geometric/nn/models/gnn_llm.py create mode 100644 torch_geometric/nn/text/__init__.py create mode 100644 torch_geometric/nn/text/llm.py create mode 100644 torch_geometric/nn/text/sentence_transformer_embedding.py diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 86f23075a6b8..3fd2b13c1686 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -21,7 +21,8 @@ from torch_geometric import seed_everything from torch_geometric.datasets import WebQSPDataset from torch_geometric.loader import DataLoader -from torch_geometric.nn.models.gnn_llm import GNN_LLM, LLM +from torch_geometric.nn.models import GRetriever +from torch_geometric.nn.text import LLM def detect_hallucinate(pred, label): @@ -99,6 +100,23 @@ def load_params_dict(model, save_path): return model +def get_loss(model, batch, model_save_name): + if model_save_name == "llm": + return model(batch.question, batch.label, batch.desc) + else: + return model(batch.question, batch.x, batch.edge_index, batch.batch, + batch.ptr, batch.label, batch.edge_attr, batch.desc) + + +def inference_step(model, batch, model_save_name): + if model_save_name == "llm": + return model.inference(batch.question, batch.desc) + else: + return model.inference(batch.question, batch.x, batch.edge_index, + batch.batch, batch.ptr, batch.edge_attr, + batch.desc) + + def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, model=None, dataset=None, checkpointing=False): def adjust_learning_rate(param_group, LR, epoch): @@ -170,7 +188,7 @@ def adjust_learning_rate(param_group, LR, epoch): loader = tqdm(train_loader, desc=epoch_str) for step, batch in enumerate(loader): optimizer.zero_grad() - loss = model(batch) + loss = get_loss(model, batch, model_save_name) loss.backward() clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) @@ -192,7 +210,7 @@ def adjust_learning_rate(param_group, LR, epoch): model.eval() with torch.no_grad(): for step, batch in enumerate(val_loader): - loss = model(batch) + loss = get_loss(model, batch, model_save_name) val_loss += loss.item() val_loss = val_loss / len(val_loader) print(epoch_str + f", Val Loss: {val_loss}") @@ -215,7 +233,8 @@ def adjust_learning_rate(param_group, LR, epoch): print("Final Evaluation...") for step, batch in enumerate(test_loader): with torch.no_grad(): - output = model.inference(batch) + output = inference_step(model, batch, model_save_name) + output["label"] = batch.label eval_output.append(output) progress_bar_test.update(1) @@ -263,7 +282,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, question = batch.question[0] correct_answer = batch.label[0] # GNN+LLM only using 32 tokens to answer, give untrained LLM more - pure_llm_out = pure_llm.inference(batch, max_out_tokens=256) + pure_llm_out = pure_llm.inference(batch.question, batch.desc, + max_out_tokens=256) gnn_llm_pred = gnn_llm_preds[i] pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0e52a1538575..58aefb012947 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,29 +1,25 @@ -from typing import Dict, List, Tuple, Union, no_type_check +from typing import Dict, List, Tuple, no_type_check import numpy as np try: + import pandas as pd from pandas import DataFrame as df WITH_PANDAS = True except ImportError as e: # noqa df = None WITH_PANDAS = False import torch -import torch.nn.functional as F try: from pcst_fast import pcst_fast WITH_PCST = True except ImportError as e: # noqa WITH_PCST = False -from torch.utils.data import DataLoader from tqdm import tqdm -try: - from transformers import AutoModel, AutoTokenizer - WITH_TRANSFORMERS = True -except ImportError as e: # noqa - WITH_TRANSFORMERS = False +from torch_geometric.nn.text import SentenceTransformer, text2embedding + try: import datasets WITH_DATASETS = True @@ -140,94 +136,6 @@ def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, return data, desc -class Dataset(torch.utils.data.Dataset): - def __init__(self, input_ids: torch.Tensor, - attention_mask: torch.Tensor) -> None: - super().__init__() - self.data = { - "input_ids": input_ids, - "att_mask": attention_mask, - } - - def __len__(self) -> int: - return self.data["input_ids"].size(0) - - def __getitem__( - self, index: Union[int, torch.Tensor]) -> Dict[str, torch.Tensor]: - if isinstance(index, torch.Tensor): - index = index.item() - batch_data = dict() - for key in self.data.keys(): - if self.data[key] is not None: - batch_data[key] = self.data[key][index] - return batch_data - - -class SentenceTransformer(torch.nn.Module): - def __init__(self, pretrained_repo: str) -> None: - super().__init__() - print(f"inherit model weights from {pretrained_repo}") - self.bert_model = AutoModel.from_pretrained(pretrained_repo) - - def mean_pooling(self, token_embeddings: torch.Tensor, - attention_mask: torch.Tensor) -> torch.Tensor: - data_type = token_embeddings.dtype - input_mask_expanded = attention_mask.unsqueeze(-1).expand( - token_embeddings.size()).to(data_type) - return torch.sum(token_embeddings * input_mask_expanded, - 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) - - def forward(self, input_ids: torch.Tensor, - att_mask: torch.Tensor) -> torch.Tensor: - bert_out = self.bert_model(input_ids=input_ids, - attention_mask=att_mask) - - # First element of model_output contains all token embeddings - token_embeddings = bert_out[0] - sentence_embeddings = self.mean_pooling(token_embeddings, att_mask) - sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) - return sentence_embeddings - - -def sbert_text2embedding(model: SentenceTransformer, - tokenizer: torch.nn.Module, device: torch.device, - text: List[str]) -> torch.Tensor: - try: - encoding = tokenizer(text, padding=True, truncation=True, - return_tensors="pt") - dataset = Dataset(input_ids=encoding.input_ids, - attention_mask=encoding.attention_mask) - - # DataLoader - dataloader = DataLoader(dataset, batch_size=256, shuffle=False) - - # Placeholder for storing the embeddings - all_embeddings_list = [] - - # Iterate through batches - with torch.no_grad(): - - for batch in dataloader: - # Move batch to the appropriate device - batch = {key: value.to(device) for key, value in batch.items()} - - # Forward pass - embeddings = model(input_ids=batch["input_ids"], - att_mask=batch["att_mask"]) - - # Append the embeddings to the list - all_embeddings_list.append(embeddings) - - # Concatenate the embeddings from all batches - all_embeddings = torch.cat(all_embeddings_list, dim=0).cpu() - except: # noqa - print( - "SBERT text embedding failed, returning torch.zeros((0, 1024))...") - return torch.zeros((0, 1024)) - - return all_embeddings - - class WebQSPDataset(InMemoryDataset): r"""The WebQuestionsSP dataset was released as part of “The Value of Semantic Parse Labeling for Knowledge @@ -254,9 +162,6 @@ def __init__( if not WITH_PCST: missing_str_list.append('pcst_fast') missing_imports = True - if not WITH_TRANSFORMERS: - missing_str_list.append('transformers') - missing_imports = True if not WITH_DATASETS: missing_str_list.append('datasets') missing_imports = True @@ -267,7 +172,6 @@ def __init__( missing_str = ' '.join(missing_str_list) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) - self.prompt = "Please answer the given question." self.graph = None self.graph_type = "Knowledge Graph" self.model_name = "sbert" @@ -299,19 +203,15 @@ def download(self) -> None: } def process(self) -> None: - import pandas pretrained_repo = "sentence-transformers/all-roberta-large-v1" self.model = SentenceTransformer(pretrained_repo) self.model.to(self.device) self.model.eval() - self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) - self.text2embedding = sbert_text2embedding self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] # encode questions print("Encoding questions...") - q_embs = self.text2embedding(self.model, self.tokenizer, self.device, - self.questions) + q_embs = text2embedding(self.model, self.device, self.questions) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] @@ -338,12 +238,11 @@ def process(self) -> None: columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr.fillna("", inplace=True) - x = self.text2embedding(self.model, self.tokenizer, self.device, - nodes.node_attr.tolist()) + x = text2embedding(self.model, self.device, + nodes.node_attr.tolist()) # encode edges - edge_attr = self.text2embedding(self.model, self.tokenizer, - self.device, - edges.edge_attr.tolist()) + edge_attr = text2embedding(self.model, self.device, + edges.edge_attr.tolist()) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) question = f"Question: {data_i['question']}\nAnswer: " diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index b9f315fd192e..1a5f4c56b13b 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -28,7 +28,7 @@ from .pmlp import PMLP from .neural_fingerprint import NeuralFingerprint from .visnet import ViSNet -from .gnn_llm import GNN_LLM +from .g_retriever import GRetriever # Deprecated: from torch_geometric.explain.algorithm.captum import (to_captum_input, captum_output_to_dicts) @@ -74,5 +74,5 @@ 'PMLP', 'NeuralFingerprint', 'ViSNet', - 'GNN_LLM', + 'GRetriever', ] diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py new file mode 100644 index 000000000000..99e65272f1d6 --- /dev/null +++ b/torch_geometric/nn/models/g_retriever.py @@ -0,0 +1,310 @@ +from typing import List, Optional + +import torch +import torch.nn as nn + +from torch_geometric.nn.models import GAT +from torch_geometric.nn.text import LLM +from torch_geometric.nn.text.llm import ( + EOS, + IGNORE_INDEX, + max_new_tokens, + max_txt_len, +) +from torch_geometric.utils import scatter + + +class GRetriever(nn.Module): + r"""This GNN+LLM implementation is based on G-retriever. + Original Paper: `_. + See `examples/llm_plus_gnn/g_retriever.py` for example usage. + + Args: + llm_to_use (str): A string representing the huggingface model you + want to use. This module has been tested for 'llama2' and 'gemma'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on + https://github.com/pyg-team/pytorch_geometric + and assign to puririshi98. (default: :obj:'llama2') + llm_use_lora (bool): use LORA from peft for training the LLM. see + https://huggingface.co/docs/peft/en/index for details. + llm_dtype (torch.dtype): The dtype to use for the LLM. + (default :obj: `torch.bloat16`) + num_llm_params (int): An integer representing how many params your + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the + available GPU memory of your GPUs (default :obj:`7`) + gnn_to_use (BasicGNN): Please pass a valid model that extends + torch_geometric.nn.models.basic_gnn.BasicGNN. (default: :obj:`GAT`) + gnn_in_channels (int): (default: 1024) + gnn_hidden_channels (int): (default: 1024) + gnn_out_channels (int): (default: 1024) + num_gnn_layers (int): (default: 4) + num_gnn_heads (int): Number of heads to use for BasicGNNs with the + `heads` kwarg. (default: 4) + mlp_hidden_dim (int): (default: 2048) + mlp_out_dim (int): (default: 4096) + """ + def __init__( + self, + llm_to_use='llama2', + llm_use_lora: bool = False, + llm_dtype=torch.bfloat16, + num_llm_params: int = 7, + gnn_to_use=GAT, + gnn_in_channels: int = 1024, + gnn_hidden_channels: int = 1024, + gnn_out_channels: int = 1024, + num_gnn_layers: int = 4, + num_gnn_heads: int = 4, + mlp_hidden_dim: int = 2048, + mlp_out_dim: int = 4096, + ): + super().__init__() + if 'llama' in llm_to_use.lower(): + self.llm_to_use = LLM('llama2', llm_dtype) + elif 'gemma' in llm_to_use.lower(): + self.llm_to_use = LLM('gemma', llm_dtype) + else: + self.llm_to_use = LLM(llm_to_use, llm_dtype) + self.llm = self.llm_to_use.llm + self.llm_dtype = llm_dtype + if llm_use_lora: + from peft import ( + LoraConfig, + get_peft_model, + prepare_model_for_kbit_training, + ) + print("Training our LLM with LORA!") + self.llm = prepare_model_for_kbit_training(self.llm) + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + self.llm = get_peft_model(self.llm, config) + self.llm_device = self.llm_to_use.llm_device + self.tokenizer = self.llm_to_use.tokenizer + print('Finished loading LLAMA!') + + self.graph_encoder = gnn_to_use( + in_channels=gnn_in_channels, + out_channels=gnn_out_channels, + hidden_channels=gnn_hidden_channels, + num_layers=num_gnn_layers, + heads=num_gnn_heads, + norm='batch_norm', + ).to(self.llm_device) + # For the MLP Projection + mlp_hidden_dim = gnn_out_channels + self.projector = nn.Sequential( + nn.Linear(gnn_out_channels, mlp_hidden_dim), + nn.Sigmoid(), + nn.Linear(mlp_hidden_dim, mlp_out_dim), + ).to(self.llm_device) + + self.word_embedding = self.llm_to_use.word_embedding + + def encode_graphs(self, node_feat, edge_index, edge_attr, batch): + x = node_feat.to(self.llm_device) + edge_index = edge_index.long().to(self.llm_device) + edge_attr = edge_attr.to(self.llm_device) + n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) + batch = batch.to(self.llm_device) + # mean pooling + g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') + return g_embeds + + def forward(self, question: List[str], node_feat: torch.Tensor, + edge_index: torch.Tensor, batch: torch.Tensor, + ptr: torch.Tensor, label: List[str], + edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None): + r"""Forward pass. + + Args: + question (List[str]): The questions/prompts. + x (torch.Tensor): The input node features. + edge_index (torch.Tensor or SparseTensor): The edge indices. + batch (torch.Tensor): The batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns + each element to a specific example. + ptr (torch.Tensor): The pointer vector, denoting the + boundaries between examples in the batch. + label (List[str]): The answers/labels. + edge_attr (torch.Tensor, optional): The edge features (if supported + by the GNN being used). (default: :obj:`None`) + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(question, additional_text_context) # noqa + # encode labels + labels = self.tokenizer(label, add_special_tokens=False) + # encode training specific special token + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + + # encode graphs + graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, + batch) + graph_embeds = self.projector(graph_embeds) + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + num_nodes_per_graph = ptr[1:] - ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[ + i][:max_new_tokens] + eos_tokens.input_ids + if additional_text_context is not None: + input_ids = context.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + else: + input_ids = questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=self.llm_dtype): + outputs = self.llm( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + return outputs.loss + + @torch.no_grad() + def inference(self, question: List[str], node_feat: torch.Tensor, + edge_index: torch.Tensor, batch: torch.Tensor, + ptr: torch.Tensor, edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None, + max_out_tokens: Optional[int] = max_new_tokens): + r"""Inference. + + Args: + question (List[str]): The questions/prompts. + x (torch.Tensor): The input node features. + edge_index (torch.Tensor or SparseTensor): The edge indices. + edge_weight (torch.Tensor, optional): The edge weights (if + supported by the underlying GNN layer). (default: :obj:`None`) + batch (torch.Tensor): The batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns + each element to a specific example. + ptr (torch.Tensor): The pointer vector, denoting the + boundaries between examples in the batch. + edge_attr (torch.Tensor, optional): The edge features (if supported + by the GNN being used). (default: :obj:`None`) + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + max_out_tokens (int, optional): How many tokens for the LLM to + generate. (default: {32}) + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(question, additional_text_context) # noqa + # encode graphs + graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, + batch) + graph_embeds = self.projector(graph_embeds) + + batch_inputs_embeds = [] + batch_attention_mask = [] + num_nodes_per_graph = ptr[1:] - ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + if additional_text_context is not None: + input_ids = context.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + else: + input_ids = questions.input_ids[i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + + with torch.cuda.amp.autocast(dtype=self.llm_dtype): + outputs = self.llm.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_new_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return { + 'pred': pred, + 'question': question, + 'desc': additional_text_context, + } + + def print_trainable_params(self): + trainable_params = 0 + all_param = 0 + for _, param in self.named_parameters(): + num_params = param.numel() + + all_param += num_params + if param.requires_grad: + trainable_params += num_params + + return trainable_params, all_param diff --git a/torch_geometric/nn/models/gnn_llm.py b/torch_geometric/nn/models/gnn_llm.py deleted file mode 100644 index 703d1318629f..000000000000 --- a/torch_geometric/nn/models/gnn_llm.py +++ /dev/null @@ -1,472 +0,0 @@ -import torch -import torch.nn as nn - -try: - from peft import ( - LoraConfig, - get_peft_model, - prepare_model_for_kbit_training, - ) - WITH_PEFT = True -except ImportError as e: # noqa - WITH_PEFT = False - -try: - from transformers import AutoModelForCausalLM, AutoTokenizer - WITH_TRANSFORMERS = True -except ImportError as e: # noqa - WITH_TRANSFORMERS = False - -from torch_geometric.data import Batch -from torch_geometric.nn.models import GAT -from torch_geometric.utils import scatter - -BOS = '[INST]' -EOS_USER = '[/INST]' -EOS = '[/s]' -IGNORE_INDEX = -100 -llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" -gemma_str_name = "google/gemma-7b" -max_txt_len = 512 -max_new_tokens = 32 -pad_token_id = 0 -padding_side = 'left' - - -def get_llm_kwargs(mem_needed): - assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" - avail_gpus = torch.cuda.device_count() - kwargs = { - "revision": "main", - } - max_mem_dict = {} - avail_mem_dict = {} - mem_total = 0 - gpus_2_use_4_llm = 0 - for i in range(avail_gpus): - available_mem = int(torch.cuda.mem_get_info(i)[0] // 1024**3) - mem_total += available_mem - avail_mem_dict[i] = available_mem - gpus_2_use_4_llm += 1 - # We want to use the minimum number of GPUs that LLM can fit on - # this is to minimize the need for interGPU communications - if mem_total >= mem_needed: - break - - for i in range(gpus_2_use_4_llm): - max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" - kwargs["max_memory"] = max_mem_dict - kwargs["device_map"] = "auto" - return kwargs - - -class LLM(nn.Module): - """This module wraps a HuggingFace Transformer based model in such - a way that makes it easy to use with PyG GNNs. - model_name (str): A string representing the huggingface model you - want to use. This module has been tested for 'llama2' and 'gemma'. - Other huggingface transformer models should work if you pass the - correct name, see huggingface.co for details. If any issues occur - please file an issue on - https://github.com/pyg-team/pytorch_geometric - and assign to puririshi98. (default: :obj:'llama2') - dtype (torch.dtype): The dtype to use for the LLM. - (default :obj: `torch.bloat16`) - num_params (int): An integer representing how many params your - huggingface transformer model has, in billions. This is used to - automatically allocate the number of gpus needed, given the - available GPU memory of your GPUs (default :obj:`7`) - - """ - def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, - num_params: int = 7): - super().__init__() - if model_name == "llama2": - self.printable_llm_name = "LLAMA2" - self.huggingface_str = llama2_str_name - elif model_name == "gemma": - self.printable_llm_name = "GEMMA" - self.huggingface_str = gemma_str_name - else: - self.printable_llm_name = llm_name - self.huggingface_str = llm_name - """ - This is a rough hueristic: - We found that LLAMA2 (7B) + GAT hits OOM - on a single 80GB GPU, but can fit on a single - GPU that is slightly larger GPU. - """ - self.mem_needed = 85 * num_params / 7 - self.llm_dtype = dtype - print('Loading ' + str(self.printable_llm_name)) - kwargs = get_llm_kwargs(self.mem_needed) - print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) - self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, - use_fast=False) - self.tokenizer.pad_token_id = pad_token_id - self.tokenizer.padding_side = padding_side - self.llm = AutoModelForCausalLM.from_pretrained( - self.huggingface_str, torch_dtype=self.llm_dtype, - low_cpu_mem_usage=True, **kwargs) - self.llm_device = self.llm.device - self.word_embedding = self.llm.model.get_input_embeddings() - - def encode_inputs(self, samples: Batch): - batch_size = len(samples['question']) - questions = self.tokenizer(samples["question"], - add_special_tokens=False) - descriptions = self.tokenizer(samples["desc"], - add_special_tokens=False) - - # encode special tokens - eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) - bos_embeds = self.word_embedding( - self.tokenizer(BOS, add_special_tokens=False, - return_tensors='pt').input_ids[0].to( - self.llm_device)) - pad_embeds = self.word_embedding( - torch.tensor(self.tokenizer.pad_token_id).to( - self.llm_device)).unsqueeze(0) - return (batch_size, questions, descriptions, eos_user_tokens, - bos_embeds, pad_embeds) - - def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, \ - bos_embeds, pad_embeds = self.encode_inputs(samples) - # encode labels - labels = self.tokenizer(samples.label, add_special_tokens=False) - # encode training specific special token - eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - - batch_inputs_embeds = [] - batch_attention_mask = [] - batch_label_input_ids = [] - for i in range(batch_size): - # Add bos & eos token - label_input_ids = labels.input_ids[ - i][:max_new_tokens] + eos_tokens.input_ids - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids + label_input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - to_cat = [bos_embeds] - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - label_input_ids = [IGNORE_INDEX - ] * (inputs_embeds.shape[0] - - len(label_input_ids)) + label_input_ids - batch_label_input_ids.append(label_input_ids) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - batch_label_input_ids[ - i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) - - with torch.cuda.amp.autocast(dtype=self.llm_dtype): - outputs = self.llm( - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - return_dict=True, - labels=label_input_ids, - ) - return outputs.loss - - def inference(self, samples: Batch, max_out_tokens=max_new_tokens): - batch_size, questions, descriptions, eos_user_tokens, \ - bos_embeds, pad_embeds = self.encode_inputs(samples) - batch_inputs_embeds = [] - batch_attention_mask = [] - for i in range(batch_size): - # Add bos & eos token - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat([bos_embeds, inputs_embeds], dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - - with torch.cuda.amp.autocast(dtype=self.llm_dtype): - outputs = self.llm.generate( - inputs_embeds=inputs_embeds, - max_new_tokens=max_out_tokens, - attention_mask=attention_mask, - # do_sample=True, - use_cache=True # IMPORTANT! - ) - pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - - return { - 'pred': pred, - 'label': samples['label'], - 'question': samples['question'], - 'desc': samples['desc'], - } - - -class GRetriever(nn.Module): - r"""This GNN+LLM implementation is based on G-retriever. - Original Paper: `_. - See `examples/llm_plus_gnn/g_retriever.py` for example usage. - - Args: - llm_to_use (str): A string representing the huggingface model you - want to use. This module has been tested for 'llama2' and 'gemma'. - Other huggingface transformer models should work if you pass the - correct name, see huggingface.co for details. If any issues occur - please file an issue on - https://github.com/pyg-team/pytorch_geometric - and assign to puririshi98. (default: :obj:'llama2') - llm_use_lora (bool): use LORA from peft for training the LLM. see - https://huggingface.co/docs/peft/en/index for details. - llm_dtype (torch.dtype): The dtype to use for the LLM. - (default :obj: `torch.bloat16`) - num_llm_params (int): An integer representing how many params your - huggingface transformer model has, in billions. This is used to - automatically allocate the number of gpus needed, given the - available GPU memory of your GPUs (default :obj:`7`) - gnn_to_use (BasicGNN): Please pass a valid model that extends - torch_geometric.nn.models.basic_gnn.BasicGNN. (default: :obj:`GAT`) - gnn_in_channels (int): (default: 1024) - gnn_hidden_channels (int): (default: 1024) - gnn_out_channels (int): (default: 1024) - num_gnn_layers (int): (default: 4) - num_gnn_heads (int): Number of heads to use for BasicGNNs with the - `heads` kwarg. (default: 4) - mlp_hidden_dim (int): (default: 2048) - mlp_out_dim (int): (default: 4096) - """ - def __init__( - self, - llm_to_use='llama2', - llm_use_lora: bool = False, - llm_dtype=torch.bfloat16, - num_llm_params: int = 7, - gnn_to_use=GAT, - gnn_in_channels: int = 1024, - gnn_hidden_channels: int = 1024, - gnn_out_channels: int = 1024, - num_gnn_layers: int = 4, - num_gnn_heads: int = 4, - mlp_hidden_dim: int = 2048, - mlp_out_dim: int = 4096, - ): - super().__init__() - if not WITH_TRANSFORMERS: - raise ImportError( - "To use GRetriever, please `pip install transformers`.") - if 'llama' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2', llm_dtype) - elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma', llm_dtype) - else: - self.llm_to_use = LLM(llm_to_use, llm_dtype) - self.llm = self.llm_to_use.llm - self.llm_dtype = llm_dtype - if llm_use_lora: - if not WITH_PEFT: - raise ImportError("To use LORA, please `pip install peft`.") - print("Training our LLM with LORA!") - self.llm = prepare_model_for_kbit_training(self.llm) - lora_r: int = 8 - lora_alpha: int = 16 - lora_dropout: float = 0.05 - lora_target_modules = [ - "q_proj", - "v_proj", - ] - config = LoraConfig( - r=lora_r, - lora_alpha=lora_alpha, - target_modules=lora_target_modules, - lora_dropout=lora_dropout, - bias="none", - task_type="CAUSAL_LM", - ) - self.llm = get_peft_model(self.llm, config) - self.llm_device = self.llm_to_use.llm_device - self.tokenizer = self.llm_to_use.tokenizer - print('Finished loading LLAMA!') - - self.graph_encoder = gnn_to_use( - in_channels=gnn_in_channels, - out_channels=gnn_out_channels, - hidden_channels=gnn_hidden_channels, - num_layers=num_gnn_layers, - heads=num_gnn_heads, - norm='batch_norm', - ).to(self.llm_device) - # For the MLP Projection - mlp_hidden_dim = gnn_out_channels - self.projector = nn.Sequential( - nn.Linear(gnn_out_channels, mlp_hidden_dim), - nn.Sigmoid(), - nn.Linear(mlp_hidden_dim, mlp_out_dim), - ).to(self.llm_device) - - self.word_embedding = self.llm_to_use.word_embedding - - def encode_graphs(self, samples: Batch): - x = samples.x.to(self.llm_device) - edge_index = samples.edge_index.long().to(self.llm_device) - edge_attr = samples.edge_attr.to(self.llm_device) - n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - batch = samples.batch.to(self.llm_device) - # mean pooling - g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') - return g_embeds - - def forward(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) - # encode labels - labels = self.tokenizer(samples.label, add_special_tokens=False) - # encode training specific special token - eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - - # encode graphs - graph_embeds = self.encode_graphs(samples) - graph_embeds = self.projector(graph_embeds) - batch_inputs_embeds = [] - batch_attention_mask = [] - batch_label_input_ids = [] - num_nodes_per_graph = samples.ptr[1:] - samples.ptr[:-1] - for i in range(batch_size): - # Add bos & eos token - label_input_ids = labels.input_ids[ - i][:max_new_tokens] + eos_tokens.input_ids - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids + label_input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - to_cat = [bos_embeds] - if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - label_input_ids = [IGNORE_INDEX - ] * (inputs_embeds.shape[0] - - len(label_input_ids)) + label_input_ids - batch_label_input_ids.append(label_input_ids) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - batch_label_input_ids[ - i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) - - with torch.cuda.amp.autocast(dtype=self.llm_dtype): - outputs = self.llm( - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - return_dict=True, - labels=label_input_ids, - ) - return outputs.loss - - def inference(self, samples: Batch): - batch_size, questions, descriptions, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(samples) - # encode graphs - graph_embeds = self.encode_graphs(samples) - graph_embeds = self.projector(graph_embeds) - - batch_inputs_embeds = [] - batch_attention_mask = [] - for i in range(batch_size): - # Add bos & eos token - input_ids = descriptions.input_ids[ - i][:max_txt_len] + questions.input_ids[ - i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat( - [bos_embeds, graph_embeds[i].unsqueeze(0), inputs_embeds], - dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - - with torch.cuda.amp.autocast(dtype=self.llm_dtype): - outputs = self.llm.generate( - inputs_embeds=inputs_embeds, - max_new_tokens=max_new_tokens, - attention_mask=attention_mask, - # do_sample=True, - use_cache=True # IMPORTANT! - ) - pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - - return { - 'pred': pred, - 'label': samples['label'], - 'question': samples['question'], - 'desc': samples['desc'], - } - - def print_trainable_params(self): - trainable_params = 0 - all_param = 0 - for _, param in self.named_parameters(): - num_params = param.numel() - - all_param += num_params - if param.requires_grad: - trainable_params += num_params - - return trainable_params, all_param diff --git a/torch_geometric/nn/text/__init__.py b/torch_geometric/nn/text/__init__.py new file mode 100644 index 000000000000..7d2d18a3ba2f --- /dev/null +++ b/torch_geometric/nn/text/__init__.py @@ -0,0 +1,10 @@ +r"""Model package.""" + +from .llm import LLM +from .sentence_transformer_embedding import SentenceTransformer, text2embedding + +__all__ = classes = [ + 'LLM', + 'SentenceTransformer', + 'text2embedding', +] diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py new file mode 100644 index 000000000000..b140bde5d22b --- /dev/null +++ b/torch_geometric/nn/text/llm.py @@ -0,0 +1,243 @@ +from typing import List, Optional + +import torch +import torch.nn as nn + +BOS = '[INST]' +EOS_USER = '[/INST]' +EOS = '[/s]' +IGNORE_INDEX = -100 +llama2_str_name = "meta-llama/Llama-2-7b-chat-hf" +gemma_str_name = "google/gemma-7b" +max_txt_len = 512 +max_new_tokens = 32 +pad_token_id = 0 +padding_side = 'left' + + +def get_llm_kwargs(mem_needed): + assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" + avail_gpus = torch.cuda.device_count() + kwargs = { + "revision": "main", + } + max_mem_dict = {} + avail_mem_dict = {} + mem_total = 0 + gpus_2_use_4_llm = 0 + for i in range(avail_gpus): + available_mem = int(torch.cuda.mem_get_info(i)[0] // 1024**3) + mem_total += available_mem + avail_mem_dict[i] = available_mem + gpus_2_use_4_llm += 1 + # We want to use the minimum number of GPUs that LLM can fit on + # this is to minimize the need for interGPU communications + if mem_total >= mem_needed: + break + + for i in range(gpus_2_use_4_llm): + max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" + kwargs["max_memory"] = max_mem_dict + kwargs["device_map"] = "auto" + return kwargs + + +class LLM(nn.Module): + r"""This module wraps a HuggingFace Transformer based model. + model_name (str): A string representing the huggingface model you + want to use. This module has been tested for 'llama2' and 'gemma'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on + https://github.com/pyg-team/pytorch_geometric + and assign to puririshi98. (default: :obj:'llama2') + dtype (torch.dtype): The dtype to use for the LLM. + (default :obj: `torch.bloat16`) + num_params (int): An integer representing how many params your + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the + available GPU memory of your GPUs (default :obj:`7`) + + """ + def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, + num_params: int = 7): + super().__init__() + from transformers import AutoModelForCausalLM, AutoTokenizer + if model_name == "llama2": + self.printable_llm_name = "LLAMA2" + self.huggingface_str = llama2_str_name + elif model_name == "gemma": + self.printable_llm_name = "GEMMA" + self.huggingface_str = gemma_str_name + else: + self.printable_llm_name = model_name + self.huggingface_str = model_name + """ + This is a rough hueristic: + We found that LLAMA2 (7B) + GAT hits OOM + on a single 80GB GPU, but can fit on a single + GPU that is slightly larger GPU. + """ + self.mem_needed = 85 * num_params / 7 + self.llm_dtype = dtype + print('Loading ' + str(self.printable_llm_name)) + kwargs = get_llm_kwargs(self.mem_needed) + print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) + self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, + use_fast=False) + self.tokenizer.pad_token_id = pad_token_id + self.tokenizer.padding_side = padding_side + self.llm = AutoModelForCausalLM.from_pretrained( + self.huggingface_str, torch_dtype=self.llm_dtype, + low_cpu_mem_usage=True, **kwargs) + self.llm_device = self.llm.device + self.word_embedding = self.llm.model.get_input_embeddings() + + def encode_inputs(self, question, additional_context=None): + batch_size = len(question) + questions = self.tokenizer(question, add_special_tokens=False) + if additional_context is not None: + additional_context = self.tokenizer(additional_context, + add_special_tokens=False) + + # encode special tokens + eos_user_tokens = self.tokenizer(EOS_USER, add_special_tokens=False) + bos_embeds = self.word_embedding( + self.tokenizer(BOS, add_special_tokens=False, + return_tensors='pt').input_ids[0].to( + self.llm_device)) + pad_embeds = self.word_embedding( + torch.tensor(self.tokenizer.pad_token_id).to( + self.llm_device)).unsqueeze(0) + return (batch_size, questions, additional_context, eos_user_tokens, + bos_embeds, pad_embeds) + + def forward(self, question: List[str], label: List[str], + additional_context: Optional[List[str]] = None): + r"""Forward pass. + + Args: + question (List[str]): The questions/prompts. + label (List[str]): The answers/labels. + additional_context (List[str], optional): Additional context to + give to the LLM, such as textified knowledge graphs. + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.encode_inputs(question, additional_context) # noqa + # encode labels + labels = self.tokenizer(label, add_special_tokens=False) + # encode training specific special token + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[ + i][:max_new_tokens] + eos_tokens.input_ids + if context is not None: + input_ids = context.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + else: + input_ids = questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.llm_device) + + with torch.cuda.amp.autocast(dtype=self.llm_dtype): + outputs = self.llm( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + return outputs.loss + + @torch.no_grad() + def inference(self, question: List[str], + additional_context: Optional[List[str]] = None, + max_out_tokens: Optional[int] = max_new_tokens): + r"""Inference. + + Args: + question (List[str]): The questions/prompts. + additional_context (List[str], optional): Additional context to + give to the LLM, such as textified knowledge graphs. + max_out_tokens (int, optional): How many tokens for the LLM to + generate. (default: {32}) + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.encode_inputs(question, additional_context) # noqa + batch_inputs_embeds = [] + batch_attention_mask = [] + for i in range(batch_size): + # Add bos & eos token + if context is not None: + input_ids = context.input_ids[ + i][:max_txt_len] + questions.input_ids[ + i] + eos_user_tokens.input_ids + else: + input_ids = questions.input_ids[i] + eos_user_tokens.input_ids + + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + inputs_embeds = torch.cat([bos_embeds, inputs_embeds], dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + + with torch.cuda.amp.autocast(dtype=self.llm_dtype): + outputs = self.llm.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_out_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + return { + 'pred': pred, + 'question': question, + 'desc': additional_context, + } diff --git a/torch_geometric/nn/text/sentence_transformer_embedding.py b/torch_geometric/nn/text/sentence_transformer_embedding.py new file mode 100644 index 000000000000..180a8d1c55e8 --- /dev/null +++ b/torch_geometric/nn/text/sentence_transformer_embedding.py @@ -0,0 +1,71 @@ +from typing import List, Optional + +import torch +import torch.nn.functional as F + + +class SentenceTransformer(torch.nn.Module): + def __init__(self, pretrained_repo: str) -> None: + super().__init__() + print(f"inherit model weights from {pretrained_repo}") + from transformers import AutoModel, AutoTokenizer + self.bert_model = AutoModel.from_pretrained(pretrained_repo) + self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) + + def mean_pooling(self, token_embeddings: torch.Tensor, + attention_mask: torch.Tensor) -> torch.Tensor: + data_type = token_embeddings.dtype + input_mask_expanded = attention_mask.unsqueeze(-1).expand( + token_embeddings.size()).to(data_type) + return torch.sum(token_embeddings * input_mask_expanded, + 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + def forward(self, input_ids: torch.Tensor, + att_mask: torch.Tensor) -> torch.Tensor: + bert_out = self.bert_model(input_ids=input_ids, + attention_mask=att_mask) + + # First element of model_output contains all token embeddings + token_embeddings = bert_out[0] + sentence_embeddings = self.mean_pooling(token_embeddings, att_mask) + sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) + return sentence_embeddings + + +def text2embedding(model: SentenceTransformer, device: torch.device, + text: List[str], + batch_size: Optional[int] = 256) -> torch.Tensor: + try: + encoding = model.tokenizer(text, padding=True, truncation=True, + return_tensors="pt") + data_len = encoding.input_ids.size(0) + num_full_batches = data_len // batch_size + all_embeddings_list = [] + + # Iterate through batches + with torch.no_grad(): + left_ptr = 0 + for i in range(num_full_batches): + # Forward pass + embeddings = model( + input_ids=encoding.input_ids[left_ptr:left_ptr + + batch_size].to(device), + att_mask=encoding.attention_mask[left_ptr:left_ptr + + batch_size].to(device)) + left_ptr += batch_size + # Append the embeddings to the list + all_embeddings_list.append(embeddings) + # final batch if len not divisible by batch_size + if data_len % batch_size != 0: + embeddings = model( + input_ids=encoding.input_ids[left_ptr:].to(device), + att_mask=encoding.attention_mask[left_ptr:].to(device)) + all_embeddings_list.append(embeddings) + # Concatenate the embeddings from all batches + all_embeddings = torch.cat(all_embeddings_list, dim=0).cpu() + except: # noqa + print( + "SBERT text embedding failed, returning torch.zeros((0, 1024))...") + return torch.zeros((0, 1024)) + + return all_embeddings From 42e84e4ea23b5f576156bc9d317b6c3ecfc7312a Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Apr 2024 08:09:15 -0700 Subject: [PATCH 369/752] cleanup --- torch_geometric/nn/models/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 99e65272f1d6..b3ad9dd9643e 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -135,7 +135,7 @@ def forward(self, question: List[str], node_feat: torch.Tensor, Args: question (List[str]): The questions/prompts. - x (torch.Tensor): The input node features. + node_feat (torch.Tensor): The input node features. edge_index (torch.Tensor or SparseTensor): The edge indices. batch (torch.Tensor): The batch vector :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns @@ -224,7 +224,7 @@ def inference(self, question: List[str], node_feat: torch.Tensor, Args: question (List[str]): The questions/prompts. - x (torch.Tensor): The input node features. + node_feat (torch.Tensor): The input node features. edge_index (torch.Tensor or SparseTensor): The edge indices. edge_weight (torch.Tensor, optional): The edge weights (if supported by the underlying GNN layer). (default: :obj:`None`) From 26752a7e76d06be3b1fc4c89085317b7f74bc76d Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Apr 2024 08:10:03 -0700 Subject: [PATCH 370/752] cleanup --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index b140bde5d22b..3f2848e03528 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -44,6 +44,7 @@ def get_llm_kwargs(mem_needed): class LLM(nn.Module): r"""This module wraps a HuggingFace Transformer based model. + model_name (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. Other huggingface transformer models should work if you pass the @@ -57,7 +58,6 @@ class LLM(nn.Module): huggingface transformer model has, in billions. This is used to automatically allocate the number of gpus needed, given the available GPU memory of your GPUs (default :obj:`7`) - """ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, num_params: int = 7): From 55969aa6c6ccb2b54f462632c611467e3210c0cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:11:16 +0000 Subject: [PATCH 371/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 3f2848e03528..e1079b5584b5 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -44,7 +44,7 @@ def get_llm_kwargs(mem_needed): class LLM(nn.Module): r"""This module wraps a HuggingFace Transformer based model. - + model_name (str): A string representing the huggingface model you want to use. This module has been tested for 'llama2' and 'gemma'. Other huggingface transformer models should work if you pass the From be61c2b87561fdfc5b5f670a65f42d1edc6a272f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 1 May 2024 13:05:51 -0700 Subject: [PATCH 372/752] cleanup --- torch_geometric/datasets/web_qsp_dataset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 58aefb012947..0ad5ffbb96ff 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -172,9 +172,6 @@ def __init__( missing_str = ' '.join(missing_str_list) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) - self.graph = None - self.graph_type = "Knowledge Graph" - self.model_name = "sbert" self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) From ba6115274bb17dbd3cfd70e3aef62d07895b2f9f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 7 May 2024 15:55:54 -0700 Subject: [PATCH 373/752] fixing a warning in latest pandas /usr/local/lib/python3.10/dist-packages/torch_geometric/datasets/web_qsp_dataset.py:237: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method. The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy. For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object. nodes.node_attr.fillna("", inplace=True) --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 0ad5ffbb96ff..86f1cbf9d2a3 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -234,7 +234,8 @@ def process(self) -> None: edges = pd.DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes - nodes.node_attr.fillna("", inplace=True) + nodes.node_attr = nodes.node_attr.fillna("") + df[col] = df[col].method(value) x = text2embedding(self.model, self.device, nodes.node_attr.tolist()) # encode edges From 42be44d6c5969c801b5e55a781f5f61ea39bd378 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 7 May 2024 15:58:45 -0700 Subject: [PATCH 374/752] cleanup --- torch_geometric/datasets/web_qsp_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 86f1cbf9d2a3..519484737e19 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -235,7 +235,6 @@ def process(self) -> None: columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") - df[col] = df[col].method(value) x = text2embedding(self.model, self.device, nodes.node_attr.tolist()) # encode edges From 3e1d36c8dd9b3f8993c6e8e6b17655835a8190f2 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 8 May 2024 10:31:09 -0700 Subject: [PATCH 375/752] minor cleanup for the demo --- examples/llm_plus_gnn/g_retriever.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3fd2b13c1686..9d5759395bdc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -2,7 +2,6 @@ Original Paper: https://arxiv.org/abs/2402.07630 “G-Retriever significantly reduces hallucinations by 54% compared to the [LLM] baseline“. - requirements on top of basic PyG: pip install datasets transformers pcst_fast sentencepiece tqdm pandas """ @@ -262,9 +261,9 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") - print("Would you like to reuse them?") + print("Would you like to redo untuned LLM eval?") user_input = str(input("(y/n):")).lower() - skip_step_one = user_input == "y" + skip_step_one = user_input == "n" else: skip_step_one = False From b7d17ac742be274611c4d0f97d491e95f524bbf7 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 9 May 2024 18:17:30 -0700 Subject: [PATCH 376/752] typo fix --- examples/llm_plus_gnn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 5c7900479540..33d9c22aa56b 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -2,4 +2,4 @@ | Example | Description | | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retriever Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | From 38a08d740331ddabd27925f36a6ad79e3c8953c9 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 14 May 2024 02:14:21 +0000 Subject: [PATCH 377/752] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c10279e38583..c497fb8628c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added -- Added the `WebQSPDataset` dataset with example training G-Retriever (GNN+LLM) ([#8984](https://github.com/pyg-team/pytorch_geometric/pull/8984)) +- Added the `WebQSPDataset` dataset with example training G-Retriever (GNN+LLM) ([#8984](https://github.com/pyg-team/pytorch_geometric/pull/9167)) - Added support for `EdgeIndex.unbind` ([#9298](https://github.com/pyg-team/pytorch_geometric/pull/9298)) - Integrate `torch_geometric.Index` into `torch_geometric.EdgeIndex` ([#9296](https://github.com/pyg-team/pytorch_geometric/pull/9296)) - Support `EdgeIndex.sparse_narrow` for non-sorted edge indices ([#9291](https://github.com/pyg-team/pytorch_geometric/pull/9291)) From 408efae58ff61769cd677f950ddf4a698aaf6e56 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 14 May 2024 02:50:02 +0000 Subject: [PATCH 378/752] formatting --- examples/llm_plus_gnn/g_retriever.py | 4 +- torch_geometric/datasets/web_qsp_dataset.py | 49 ++++++++++----------- torch_geometric/nn/models/g_retriever.py | 36 ++++++++++----- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9d5759395bdc..2dc203f0b4ae 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -36,7 +36,7 @@ def detect_hallucinate(pred, label): return "skip" -def compute_accuracy(eval_output): +def compute_accuracy(eval_output) -> float: df = pd.concat([pd.DataFrame(d) for d in eval_output]) all_hit = [] all_precision = [] @@ -99,7 +99,7 @@ def load_params_dict(model, save_path): return model -def get_loss(model, batch, model_save_name): +def get_loss(model, batch, model_save_name) -> torch.Tensor: if model_save_name == "llm": return model(batch.question, batch.label, batch.desc) else: diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 519484737e19..88d5d95d873e 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,41 +1,44 @@ +# Alot of code in this file is based on the original G-Retriever paper +# url: https://arxiv.org/abs/2402.07630 from typing import Dict, List, Tuple, no_type_check import numpy as np +import torch +from tqdm import tqdm + +from torch_geometric.nn.text import SentenceTransformer, text2embedding +from torch_geometric.data import Data, InMemoryDataset try: - import pandas as pd - from pandas import DataFrame as df + from pandas import DataFrame WITH_PANDAS = True -except ImportError as e: # noqa - df = None +except ImportError: + DataFrame = None WITH_PANDAS = False -import torch try: from pcst_fast import pcst_fast WITH_PCST = True -except ImportError as e: # noqa +except ImportError: WITH_PCST = False -from tqdm import tqdm - -from torch_geometric.nn.text import SentenceTransformer, text2embedding try: import datasets WITH_DATASETS = True -except ImportError as e: # noqa +except ImportError: WITH_DATASETS = False -from torch_geometric.data import Data, InMemoryDataset - -# Alot of code in this file is based on the original G-Retriever paper -# url: https://arxiv.org/abs/2402.07630 - @no_type_check -def retrieval_via_pcst(graph: Data, q_emb: torch.Tensor, textual_nodes: df, - textual_edges: df, topk: int = 3, topk_e: int = 3, - cost_e: float = 0.5) -> Tuple[Data, str]: +def retrieval_via_pcst( + graph: Data, + q_emb: torch.Tensor, + textual_nodes: DataFrame, + textual_edges: DataFrame, + topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5, +) -> Tuple[Data, str]: c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( @@ -157,18 +160,14 @@ def __init__( root: str = "", force_reload: bool = False, ) -> None: - missing_imports = False missing_str_list = [] if not WITH_PCST: missing_str_list.append('pcst_fast') - missing_imports = True if not WITH_DATASETS: missing_str_list.append('datasets') - missing_imports = True if not WITH_PANDAS: missing_str_list.append('pandas') - missing_imports = True - if missing_imports: + if len(missing_str_list) > 0: missing_str = ' '.join(missing_str_list) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) @@ -227,11 +226,11 @@ def process(self) -> None: "edge_attr": r, "dst": raw_nodes[t] }) - nodes = pd.DataFrame([{ + nodes = DataFrame([{ "node_id": v, "node_attr": k } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"]) - edges = pd.DataFrame(raw_edges, + edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index b3ad9dd9643e..2af37853ead0 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -60,7 +60,7 @@ def __init__( num_gnn_heads: int = 4, mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096, - ): + ) -> None: super().__init__() if 'llama' in llm_to_use.lower(): self.llm_to_use = LLM('llama2', llm_dtype) @@ -126,11 +126,17 @@ def encode_graphs(self, node_feat, edge_index, edge_attr, batch): g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds - def forward(self, question: List[str], node_feat: torch.Tensor, - edge_index: torch.Tensor, batch: torch.Tensor, - ptr: torch.Tensor, label: List[str], - edge_attr: Optional[torch.Tensor] = None, - additional_text_context: Optional[List[str]] = None): + def forward( + self, + question: List[str], + node_feat: torch.Tensor, + edge_index: torch.Tensor, + batch: torch.Tensor, + ptr: torch.Tensor, + label: List[str], + edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None, + ): r"""Forward pass. Args: @@ -215,11 +221,17 @@ def forward(self, question: List[str], node_feat: torch.Tensor, return outputs.loss @torch.no_grad() - def inference(self, question: List[str], node_feat: torch.Tensor, - edge_index: torch.Tensor, batch: torch.Tensor, - ptr: torch.Tensor, edge_attr: Optional[torch.Tensor] = None, - additional_text_context: Optional[List[str]] = None, - max_out_tokens: Optional[int] = max_new_tokens): + def inference( + self, + question: List[str], + node_feat: torch.Tensor, + edge_index: torch.Tensor, + batch: torch.Tensor, + ptr: torch.Tensor, + edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None, + max_out_tokens: Optional[int] = max_new_tokens, + ): r"""Inference. Args: @@ -297,7 +309,7 @@ def inference(self, question: List[str], node_feat: torch.Tensor, 'desc': additional_text_context, } - def print_trainable_params(self): + def print_trainable_params(self) -> None: trainable_params = 0 all_param = 0 for _, param in self.named_parameters(): From ac7fcdba1ba326799efdd505fd9bcf7490d7e334 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 02:52:08 +0000 Subject: [PATCH 379/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 88d5d95d873e..27d6b459c493 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -6,8 +6,8 @@ import torch from tqdm import tqdm -from torch_geometric.nn.text import SentenceTransformer, text2embedding from torch_geometric.data import Data, InMemoryDataset +from torch_geometric.nn.text import SentenceTransformer, text2embedding try: from pandas import DataFrame @@ -230,8 +230,7 @@ def process(self) -> None: "node_id": v, "node_attr": k } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"]) - edges = DataFrame(raw_edges, - columns=["src", "edge_attr", "dst"]) + edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") x = text2embedding(self.model, self.device, From 19bcadab791c4166f9c1fe84810ebbe1f0ce2c9f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 14 May 2024 15:26:44 -0700 Subject: [PATCH 380/752] Update g_retriever.py --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 2dc203f0b4ae..f8aa6399a25c 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -3,7 +3,7 @@ “G-Retriever significantly reduces hallucinations by 54% compared to the [LLM] baseline“. requirements on top of basic PyG: -pip install datasets transformers pcst_fast sentencepiece tqdm pandas +pip install datasets transformers pcst_fast sentencepiece tqdm pandas accelerate """ import argparse import gc From 22a018403ed06f0c691a21c4a206a773fd531454 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 15:12:02 +0000 Subject: [PATCH 381/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 4953387e9055..af324f09f79d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -20,4 +20,4 @@ For examples on using `torch.compile`, see examples and README under [`examples/ For examples on working with heterogeneous data, see the examples under [`examples/hetero`](./hetero). -For examples on co-training LLMs with GNNs, see the examples under [`examples/llm`](./llm). \ No newline at end of file +For examples on co-training LLMs with GNNs, see the examples under [`examples/llm`](./llm). From 21a7ddcdbfd476e2b78ccb5b6265939d5ec40331 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 12:44:37 -0700 Subject: [PATCH 382/752] accomadating smaller gpus w/ cpu offloding --- test/nn/nlp/test_llm.py | 0 torch_geometric/nn/text/llm.py | 13 ++++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 test/nn/nlp/test_llm.py diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index e1079b5584b5..3e8bcd729c7d 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -34,11 +34,14 @@ def get_llm_kwargs(mem_needed): # this is to minimize the need for interGPU communications if mem_total >= mem_needed: break - - for i in range(gpus_2_use_4_llm): - max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" - kwargs["max_memory"] = max_mem_dict - kwargs["device_map"] = "auto" + if mem_total >= mem_needed: + for i in range(gpus_2_use_4_llm): + max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" + kwargs["max_memory"] = max_mem_dict + kwargs["device_map"] = "auto" + else: + kwargs["device_map"] = "auto" + kwargs["cpu_offload"] = True return kwargs From 27cc851c504869728110693a6ac6651cae5dbccc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:05:49 -0700 Subject: [PATCH 383/752] accomadating smaller gpus w/ cpu offloding --- torch_geometric/nn/text/llm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 3e8bcd729c7d..52179b2a8d49 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -39,10 +39,11 @@ def get_llm_kwargs(mem_needed): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict kwargs["device_map"] = "auto" + cpu_offload = False else: kwargs["device_map"] = "auto" - kwargs["cpu_offload"] = True - return kwargs + cpu_offload = True + return kwargs, cpu_offload class LLM(nn.Module): @@ -84,7 +85,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.mem_needed = 85 * num_params / 7 self.llm_dtype = dtype print('Loading ' + str(self.printable_llm_name)) - kwargs = get_llm_kwargs(self.mem_needed) + kwargs, cpu_offload = get_llm_kwargs(self.mem_needed) print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, use_fast=False) @@ -93,7 +94,11 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, torch_dtype=self.llm_dtype, low_cpu_mem_usage=True, **kwargs) + self.llm_device = self.llm.device + if cpu_offload: + import accelerate + accelerate.cpu_offload(self.llm, execution_device=self.llm_device) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From ab207ca82e3c1963a7515636080ef7b325800ae6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:06:53 +0000 Subject: [PATCH 384/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 52179b2a8d49..6fb94f88cb49 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -94,7 +94,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, torch_dtype=self.llm_dtype, low_cpu_mem_usage=True, **kwargs) - + self.llm_device = self.llm.device if cpu_offload: import accelerate From 0254435a0045c342df3abfc6d7081cbff8e06679 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:08:13 -0700 Subject: [PATCH 385/752] accomadating smaller gpus w/ cpu offloding --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 52179b2a8d49..3dd0bf9d9af3 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -98,7 +98,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = self.llm.device if cpu_offload: import accelerate - accelerate.cpu_offload(self.llm, execution_device=self.llm_device) + self.llm = accelerate.cpu_offload(self.llm, execution_device=self.llm_device) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From 323a1c364126c5f104797d34b254e40d7a55361e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:09:36 -0700 Subject: [PATCH 386/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c1a74aee8e12..357f863b127f 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -98,6 +98,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = self.llm.device if cpu_offload: import accelerate + print("turning on cpu offload") self.llm = accelerate.cpu_offload(self.llm, execution_device=self.llm_device) self.word_embedding = self.llm.model.get_input_embeddings() From 7cdf5a7ef12e6aa8aac496a1903bc62cae7437e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:12:52 +0000 Subject: [PATCH 387/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 357f863b127f..37e0db048193 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -99,7 +99,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: import accelerate print("turning on cpu offload") - self.llm = accelerate.cpu_offload(self.llm, execution_device=self.llm_device) + self.llm = accelerate.cpu_offload(self.llm, + execution_device=self.llm_device) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From af5be61d1040ed7072e810e65c6199614bca029d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:16:03 -0700 Subject: [PATCH 388/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 357f863b127f..dd88199b4fa5 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -93,7 +93,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, torch_dtype=self.llm_dtype, - low_cpu_mem_usage=True, **kwargs) + low_cpu_mem_usage=not cpu_offload, **kwargs) self.llm_device = self.llm.device if cpu_offload: From 9c53b8b11a2b0cc1a096217eca5f040792fac585 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:17:34 -0700 Subject: [PATCH 389/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 5f235b628316..fba6db081da7 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -43,6 +43,8 @@ def get_llm_kwargs(mem_needed): else: kwargs["device_map"] = "auto" cpu_offload = True + kwargs["low_cpu_mem_usage"] = not cpu_offload + return kwargs, cpu_offload @@ -92,8 +94,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( - self.huggingface_str, torch_dtype=self.llm_dtype, - low_cpu_mem_usage=not cpu_offload, **kwargs) + self.huggingface_str, torch_dtype=self.llm_dtype, **kwargs) self.llm_device = self.llm.device if cpu_offload: From 0f9c14b6b23eed32a476aab94c023c347051ffef Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:18:33 -0700 Subject: [PATCH 390/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index fba6db081da7..5ba8484ee5cf 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -41,7 +41,6 @@ def get_llm_kwargs(mem_needed): kwargs["device_map"] = "auto" cpu_offload = False else: - kwargs["device_map"] = "auto" cpu_offload = True kwargs["low_cpu_mem_usage"] = not cpu_offload From 739e7f6f690db6a91ad190ee8670ddd881510d0e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:20:50 -0700 Subject: [PATCH 391/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 5ba8484ee5cf..ffa04ea49970 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -182,7 +182,7 @@ def forward(self, question: List[str], label: List[str], label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=self.llm_dtype): + with torch.amp.autocast(dtype=self.llm_dtype): outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, From c40141a9dbc3fd31738bf4d38a392e28936c6064 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:23:31 -0700 Subject: [PATCH 392/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index ffa04ea49970..b507c4fcf648 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,7 +100,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, import accelerate print("turning on cpu offload") self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.llm_device) + execution_device=torch.device("cuda:0")) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): @@ -182,7 +182,7 @@ def forward(self, question: List[str], label: List[str], label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.amp.autocast(dtype=self.llm_dtype): + with torch.cuda.amp.autocast(dtype=self.llm_dtype): outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, From 5dc8fe325e637885a31be0c8f8d3db858124931a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:24:34 +0000 Subject: [PATCH 393/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index b507c4fcf648..4486411571fb 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -99,8 +99,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: import accelerate print("turning on cpu offload") - self.llm = accelerate.cpu_offload(self.llm, - execution_device=torch.device("cuda:0")) + self.llm = accelerate.cpu_offload( + self.llm, execution_device=torch.device("cuda:0")) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From 097cef0659f00d20f3f194da528050c78865921f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:30:05 -0700 Subject: [PATCH 394/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index b507c4fcf648..db6bd7a64db1 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -95,12 +95,14 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, torch_dtype=self.llm_dtype, **kwargs) - self.llm_device = self.llm.device + if cpu_offload: import accelerate - print("turning on cpu offload") + self.llm_device = torch.device("cuda:0") self.llm = accelerate.cpu_offload(self.llm, - execution_device=torch.device("cuda:0")) + execution_device=self.llm_device) + else: + self.llm_device = self.llm.device self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From 32e8489ca6d88b18cbe8ba2e528b4a18830ad544 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:31:29 +0000 Subject: [PATCH 395/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index db6bd7a64db1..68494d5cd760 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -95,7 +95,6 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, torch_dtype=self.llm_dtype, **kwargs) - if cpu_offload: import accelerate self.llm_device = torch.device("cuda:0") From e9ea289ab74112e1fd2c67c4f8f6034091085da7 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:35:32 -0700 Subject: [PATCH 396/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index db6bd7a64db1..a96dadee87ca 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,7 +100,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, import accelerate self.llm_device = torch.device("cuda:0") self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.llm_device) + execution_device=self.llm_device, + offload_buffers=True) else: self.llm_device = self.llm.device self.word_embedding = self.llm.model.get_input_embeddings() From 580c9124d5663ee74d66123cb423e2eb642eec9a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:37:55 -0700 Subject: [PATCH 397/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index e6f911a3f41e..ef52bf407c1f 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -97,9 +97,10 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: import accelerate - self.llm_device = torch.device("cuda:0") + self.llm_device = torch.device("cpu") + exec_dev = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.llm_device, + execution_device=exec_dev, offload_buffers=True) else: self.llm_device = self.llm.device From 09d9e842b1e5dec0732f0de71cee31f9bf35f70b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:39:05 +0000 Subject: [PATCH 398/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index ef52bf407c1f..b938883d2d9b 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -98,7 +98,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: import accelerate self.llm_device = torch.device("cpu") - exec_dev = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") + exec_dev = torch.device( + "cuda:0") if torch.cuda.is_available() else torch.device("cpu") self.llm = accelerate.cpu_offload(self.llm, execution_device=exec_dev, offload_buffers=True) From a5da038db05d17f8d3ef232a896c5ab5ed209a4e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:42:50 -0700 Subject: [PATCH 399/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2af37853ead0..a275a3fa715b 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -184,7 +184,7 @@ def forward( torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(graph_embeds[i].unsqueeze(0).to(self.llm_device)) to_cat.append(inputs_embeds) inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) From 29c15178eb92fa7172c0e809318b281e2380236c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:45:13 -0700 Subject: [PATCH 400/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index a275a3fa715b..e723031af9fc 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -184,9 +184,9 @@ def forward( torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0).to(self.llm_device)) + to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) + inputs_embeds = torch.cat([to_cat.to(self.llm_device)], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 84a4da6b16628c8972b2a4a62b439bf92a42d247 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:46:21 -0700 Subject: [PATCH 401/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index e723031af9fc..309add2ad7a5 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -186,7 +186,7 @@ def forward( if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([to_cat.to(self.llm_device)], dim=0) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 130315b621cfae0aae55f9c86af05e630bbf5aca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:48:29 +0000 Subject: [PATCH 402/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 309add2ad7a5..32812d22dd23 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -186,7 +186,8 @@ def forward( if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], dim=0) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) label_input_ids = [IGNORE_INDEX From 17ebeb775234845303112dfa710de9d9362dd28c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:49:58 -0700 Subject: [PATCH 403/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 32812d22dd23..c75d942dff91 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -277,7 +277,8 @@ def inference( if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) From 538415c7cce882fd755770de7db820e383d8e0cb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:52:14 -0700 Subject: [PATCH 404/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index c75d942dff91..7537472a565b 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -200,7 +200,7 @@ def forward( for i in range(batch_size): pad_length = max_length - batch_inputs_embeds[i].shape[0] batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + [pad_embeds.repeat(pad_length, 1).to(self.llm_device), batch_inputs_embeds[i].to(self.llm_device)]) batch_attention_mask[i] = [0 ] * pad_length + batch_attention_mask[i] batch_label_input_ids[ From f21145c0849a72f462586d0bef991fd461891879 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:53:26 +0000 Subject: [PATCH 405/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 7537472a565b..491a945c642c 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -199,8 +199,10 @@ def forward( max_length = max([x.shape[0] for x in batch_inputs_embeds]) for i in range(batch_size): pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1).to(self.llm_device), batch_inputs_embeds[i].to(self.llm_device)]) + batch_inputs_embeds[i] = torch.cat([ + pad_embeds.repeat(pad_length, 1).to(self.llm_device), + batch_inputs_embeds[i].to(self.llm_device) + ]) batch_attention_mask[i] = [0 ] * pad_length + batch_attention_mask[i] batch_label_input_ids[ From 43d4059fb3329013be2d2fde7243edff2ea4213c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:56:28 -0700 Subject: [PATCH 406/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 11 ++++++----- torch_geometric/nn/text/llm.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 7537472a565b..b498622fc5c8 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -95,6 +95,7 @@ def __init__( ) self.llm = get_peft_model(self.llm, config) self.llm_device = self.llm_to_use.llm_device + self.exec_device = self.llm_to_use.exec_device self.tokenizer = self.llm_to_use.tokenizer print('Finished loading LLAMA!') @@ -117,11 +118,11 @@ def __init__( self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, node_feat, edge_index, edge_attr, batch): - x = node_feat.to(self.llm_device) - edge_index = edge_index.long().to(self.llm_device) - edge_attr = edge_attr.to(self.llm_device) + x = node_feat.to(self.exec_device) + edge_index = edge_index.long().to(self.exec_device) + edge_attr = edge_attr.to(self.exec_device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - batch = batch.to(self.llm_device) + batch = batch.to(self.exec_device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds @@ -212,7 +213,7 @@ def forward( label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=self.llm_dtype): + with torch.autocast(dtype=self.llm_dtype): outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index b938883d2d9b..edaa02315c40 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -98,13 +98,14 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: import accelerate self.llm_device = torch.device("cpu") - exec_dev = torch.device( + self.exec_device = torch.device( "cuda:0") if torch.cuda.is_available() else torch.device("cpu") self.llm = accelerate.cpu_offload(self.llm, execution_device=exec_dev, offload_buffers=True) else: self.llm_device = self.llm.device + self.exec_device = self.llm_device self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From 608b96ce314884b6cc49013bd7f20587fec97244 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:57:29 -0700 Subject: [PATCH 407/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index edaa02315c40..dc9d44ba6b27 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -101,7 +101,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.exec_device = torch.device( "cuda:0") if torch.cuda.is_available() else torch.device("cpu") self.llm = accelerate.cpu_offload(self.llm, - execution_device=exec_dev, + execution_device=self.exec_device, offload_buffers=True) else: self.llm_device = self.llm.device From 8171247211ce991f2e75546c3efdce47985e5c19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 20:58:36 +0000 Subject: [PATCH 408/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index dc9d44ba6b27..9335ab75191d 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,9 +100,9 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") self.exec_device = torch.device( "cuda:0") if torch.cuda.is_available() else torch.device("cpu") - self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.exec_device, - offload_buffers=True) + self.llm = accelerate.cpu_offload( + self.llm, execution_device=self.exec_device, + offload_buffers=True) else: self.llm_device = self.llm.device self.exec_device = self.llm_device From ba65a6b4652fae6a5a684e89134cefead19b1ec5 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 13:59:11 -0700 Subject: [PATCH 409/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index dc9d44ba6b27..c9fdd99ba3cd 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -96,13 +96,15 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.huggingface_str, torch_dtype=self.llm_dtype, **kwargs) if cpu_offload: - import accelerate self.llm_device = torch.device("cpu") - self.exec_device = torch.device( - "cuda:0") if torch.cuda.is_available() else torch.device("cpu") - self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.exec_device, - offload_buffers=True) + if torch.cuda.is_available(): + import accelerate + self.exec_device = torch.device("cuda:0") + self.llm = accelerate.cpu_offload(self.llm, + execution_device=self.exec_device, + offload_buffers=True) + else: + self.exec_device = torch.device("cpu") else: self.llm_device = self.llm.device self.exec_device = self.llm_device From 4b861eae7fd36ea35314e233801345db17c4c04a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:00:53 -0700 Subject: [PATCH 410/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 9e47480f3dd4..ca2d6c45d410 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -106,7 +106,7 @@ def __init__( num_layers=num_gnn_layers, heads=num_gnn_heads, norm='batch_norm', - ).to(self.llm_device) + ).to(self.exec_device) # For the MLP Projection mlp_hidden_dim = gnn_out_channels self.projector = nn.Sequential( From 26068770354d713a105478d3579290cb0fff95f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 21:00:54 +0000 Subject: [PATCH 411/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c9fdd99ba3cd..86218932d718 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,9 +100,9 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if torch.cuda.is_available(): import accelerate self.exec_device = torch.device("cuda:0") - self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.exec_device, - offload_buffers=True) + self.llm = accelerate.cpu_offload( + self.llm, execution_device=self.exec_device, + offload_buffers=True) else: self.exec_device = torch.device("cpu") else: From a57e2040d4ce25f0fa903c96c5e4061928936eb0 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:01:57 -0700 Subject: [PATCH 412/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ca2d6c45d410..da097813285f 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -113,7 +113,7 @@ def __init__( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), nn.Linear(mlp_hidden_dim, mlp_out_dim), - ).to(self.llm_device) + ).to(self.exec_device) self.word_embedding = self.llm_to_use.word_embedding From e5b11d7ace9e9e5ec7fec355d4ca4314de4df1ed Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:08:51 -0700 Subject: [PATCH 413/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- torch_geometric/nn/text/llm.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index da097813285f..fd026a274a0b 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -215,7 +215,7 @@ def forward( label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.autocast(dtype=self.llm_dtype): + with self.llm.autocast_context: outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 86218932d718..6e1b1f7ceff2 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -99,15 +99,19 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") if torch.cuda.is_available(): import accelerate - self.exec_device = torch.device("cuda:0") + device_type = "cuda" + self.exec_device = torch.device(device_type) self.llm = accelerate.cpu_offload( - self.llm, execution_device=self.exec_device, + self.llm, execution_device=device_type, offload_buffers=True) else: - self.exec_device = torch.device("cpu") + device_type = "cpu" + self.exec_device = torch.device(device_type) + self.autocast_context = torch.autocast(device_type=device_type, dtype=self.llm_dtype) else: self.llm_device = self.llm.device self.exec_device = self.llm_device + self.autocast_context = torch.cuda.amp.autocast(dtype=self.llm_dtype) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): @@ -189,7 +193,7 @@ def forward(self, question: List[str], label: List[str], label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with torch.cuda.amp.autocast(dtype=self.llm_dtype): + with self.autocast_context: outputs = self.llm( inputs_embeds=inputs_embeds, attention_mask=attention_mask, From 2bcff004d946d96e2ab293e50c003d2db4fe96df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 21:09:54 +0000 Subject: [PATCH 414/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 6e1b1f7ceff2..58243afda5e2 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -101,17 +101,19 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, import accelerate device_type = "cuda" self.exec_device = torch.device(device_type) - self.llm = accelerate.cpu_offload( - self.llm, execution_device=device_type, - offload_buffers=True) + self.llm = accelerate.cpu_offload(self.llm, + execution_device=device_type, + offload_buffers=True) else: device_type = "cpu" self.exec_device = torch.device(device_type) - self.autocast_context = torch.autocast(device_type=device_type, dtype=self.llm_dtype) + self.autocast_context = torch.autocast(device_type=device_type, + dtype=self.llm_dtype) else: self.llm_device = self.llm.device self.exec_device = self.llm_device - self.autocast_context = torch.cuda.amp.autocast(dtype=self.llm_dtype) + self.autocast_context = torch.cuda.amp.autocast( + dtype=self.llm_dtype) self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): From 1044c7597af784e4a6b0987720356c54bbc0d413 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:11:14 -0700 Subject: [PATCH 415/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index fd026a274a0b..9837c32e7a7d 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -68,7 +68,7 @@ def __init__( self.llm_to_use = LLM('gemma', llm_dtype) else: self.llm_to_use = LLM(llm_to_use, llm_dtype) - self.llm = self.llm_to_use.llm + self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: from peft import ( @@ -77,7 +77,7 @@ def __init__( prepare_model_for_kbit_training, ) print("Training our LLM with LORA!") - self.llm = prepare_model_for_kbit_training(self.llm) + self.llm_generator = prepare_model_for_kbit_training(self.llm) lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 @@ -93,7 +93,7 @@ def __init__( bias="none", task_type="CAUSAL_LM", ) - self.llm = get_peft_model(self.llm, config) + self.llm_generator = get_peft_model(self.llm_generator, config) self.llm_device = self.llm_to_use.llm_device self.exec_device = self.llm_to_use.exec_device self.tokenizer = self.llm_to_use.tokenizer @@ -215,8 +215,8 @@ def forward( label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - with self.llm.autocast_context: - outputs = self.llm( + with self.llm_to_use.autocast_context: + outputs = self.llm_generator( inputs_embeds=inputs_embeds, attention_mask=attention_mask, return_dict=True, @@ -298,8 +298,8 @@ def inference( dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - with torch.cuda.amp.autocast(dtype=self.llm_dtype): - outputs = self.llm.generate( + with self.llm_to_use.autocast_context: + outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, max_new_tokens=max_new_tokens, attention_mask=attention_mask, From b8c8ca9021c6b5830f9b4019d2accee2dd2f5b42 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:15:32 -0700 Subject: [PATCH 416/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 58243afda5e2..f10378fe64d9 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -99,15 +99,13 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") if torch.cuda.is_available(): import accelerate - device_type = "cuda" - self.exec_device = torch.device(device_type) + self.exec_device = torch.device("cuda:0") self.llm = accelerate.cpu_offload(self.llm, - execution_device=device_type, + execution_device=self.exec_device, offload_buffers=True) else: - device_type = "cpu" - self.exec_device = torch.device(device_type) - self.autocast_context = torch.autocast(device_type=device_type, + self.exec_device = torch.device("cpu") + self.autocast_context = torch.autocast(device_type="cpu", dtype=self.llm_dtype) else: self.llm_device = self.llm.device From de87a8789d4fe8a49e3429032526bb2921eeb549 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 21:16:34 +0000 Subject: [PATCH 417/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index f10378fe64d9..641f74f87ee8 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,9 +100,9 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if torch.cuda.is_available(): import accelerate self.exec_device = torch.device("cuda:0") - self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.exec_device, - offload_buffers=True) + self.llm = accelerate.cpu_offload( + self.llm, execution_device=self.exec_device, + offload_buffers=True) else: self.exec_device = torch.device("cpu") self.autocast_context = torch.autocast(device_type="cpu", From 1c06e6319670b9041591d1e221ea050b39bef95e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:16:51 -0700 Subject: [PATCH 418/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index f10378fe64d9..23236f4ce2cd 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -105,8 +105,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, offload_buffers=True) else: self.exec_device = torch.device("cpu") - self.autocast_context = torch.autocast(device_type="cpu", - dtype=self.llm_dtype) + from contextlib import nullcontext + self.autocast_context = nullcontext() else: self.llm_device = self.llm.device self.exec_device = self.llm_device From 366da06195e0d6e0554cbebc13391289e5a85a6e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:22:18 -0700 Subject: [PATCH 419/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 6842c2591bad..66118d67d160 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -15,7 +15,7 @@ padding_side = 'left' -def get_llm_kwargs(mem_needed): +def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" avail_gpus = torch.cuda.device_count() kwargs = { @@ -39,11 +39,13 @@ def get_llm_kwargs(mem_needed): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict kwargs["device_map"] = "auto" + kwargs["torch_dtype"] = autocast_dtype cpu_offload = False else: cpu_offload = True kwargs["low_cpu_mem_usage"] = not cpu_offload + return kwargs, cpu_offload @@ -86,14 +88,14 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.mem_needed = 85 * num_params / 7 self.llm_dtype = dtype print('Loading ' + str(self.printable_llm_name)) - kwargs, cpu_offload = get_llm_kwargs(self.mem_needed) + kwargs, cpu_offload = get_llm_kwargs(self.mem_needed, self.llm_dtype) print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, use_fast=False) self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( - self.huggingface_str, torch_dtype=self.llm_dtype, **kwargs) + self.huggingface_str, **kwargs) if cpu_offload: self.llm_device = torch.device("cpu") @@ -247,7 +249,7 @@ def inference(self, question: List[str], dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - with torch.cuda.amp.autocast(dtype=self.llm_dtype): + with self.autocast_context: outputs = self.llm.generate( inputs_embeds=inputs_embeds, max_new_tokens=max_out_tokens, From 7f9309dfe3466063100befbf4360611ec7a6605d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 21:23:19 +0000 Subject: [PATCH 420/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 66118d67d160..72bf91001c7c 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -45,7 +45,6 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): cpu_offload = True kwargs["low_cpu_mem_usage"] = not cpu_offload - return kwargs, cpu_offload From 699de0a1641890f9ca147ddfd35c31297ac8fe3e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:31:34 -0700 Subject: [PATCH 421/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 22 +++++++++++----------- torch_geometric/nn/text/llm.py | 11 ++++------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 9837c32e7a7d..2409be1fa2cd 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -182,12 +182,12 @@ def forward( input_ids = questions.input_ids[ i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) + torch.tensor(input_ids).to(self.exec_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + inputs_embeds = torch.cat([i.to(self.exec_device) for i in to_cat], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -201,8 +201,8 @@ def forward( for i in range(batch_size): pad_length = max_length - batch_inputs_embeds[i].shape[0] batch_inputs_embeds[i] = torch.cat([ - pad_embeds.repeat(pad_length, 1).to(self.llm_device), - batch_inputs_embeds[i].to(self.llm_device) + pad_embeds.repeat(pad_length, 1).to(self.exec_device), + batch_inputs_embeds[i].to(self.exec_device) ]) batch_attention_mask[i] = [0 ] * pad_length + batch_attention_mask[i] @@ -210,10 +210,10 @@ def forward( i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + dim=0).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) + self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator( @@ -275,12 +275,12 @@ def inference( else: input_ids = questions.input_ids[i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) + torch.tensor(input_ids).to(self.exec_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + inputs_embeds = torch.cat([i.to(self.exec_device) for i in to_cat], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -295,8 +295,8 @@ def inference( ] * pad_length + batch_attention_mask[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + dim=0).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 66118d67d160..43a0fc6cdc91 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -34,17 +34,14 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): # this is to minimize the need for interGPU communications if mem_total >= mem_needed: break - if mem_total >= mem_needed: + cpu_offload mem_total < mem_needed: + if not cpu_offload: for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict kwargs["device_map"] = "auto" kwargs["torch_dtype"] = autocast_dtype - cpu_offload = False - else: - cpu_offload = True - kwargs["low_cpu_mem_usage"] = not cpu_offload - + kwargs["low_cpu_mem_usage"] = True return kwargs, cpu_offload @@ -114,7 +111,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) - self.word_embedding = self.llm.model.get_input_embeddings() + self.word_embedding = self.llm.model.get_input_embeddings().to(self.exec_device) def encode_inputs(self, question, additional_context=None): batch_size = len(question) From a0768a5e9b5f8829a40e8882116cfede21a54cec Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:32:33 -0700 Subject: [PATCH 422/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 43a0fc6cdc91..ea2ac6440a23 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -34,7 +34,7 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): # this is to minimize the need for interGPU communications if mem_total >= mem_needed: break - cpu_offload mem_total < mem_needed: + cpu_offload = mem_total < mem_needed: if not cpu_offload: for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" From 1d41aadaa54e861e99f1571e4913a8f418e20f72 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:33:09 -0700 Subject: [PATCH 423/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index ea2ac6440a23..c36fd1de84c1 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -34,7 +34,7 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): # this is to minimize the need for interGPU communications if mem_total >= mem_needed: break - cpu_offload = mem_total < mem_needed: + cpu_offload = mem_total < mem_needed if not cpu_offload: for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" From c199dac37bd172605e4a144daf995ba047e77906 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 21:34:33 +0000 Subject: [PATCH 424/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 6 ++++-- torch_geometric/nn/text/llm.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2409be1fa2cd..5cbd697293f0 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -211,7 +211,8 @@ def forward( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.exec_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.exec_device) @@ -296,7 +297,8 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c36fd1de84c1..72b1c6412d33 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -111,7 +111,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) - self.word_embedding = self.llm.model.get_input_embeddings().to(self.exec_device) + self.word_embedding = self.llm.model.get_input_embeddings().to( + self.exec_device) def encode_inputs(self, question, additional_context=None): batch_size = len(question) From 94feff0fd83c0126936b0606a10806fa52376cea Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:34:39 -0700 Subject: [PATCH 425/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 4 ++-- torch_geometric/nn/text/llm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2409be1fa2cd..212aac5af0ed 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -182,7 +182,7 @@ def forward( input_ids = questions.input_ids[ i] + eos_user_tokens.input_ids + label_input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.exec_device)) + torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) @@ -275,7 +275,7 @@ def inference( else: input_ids = questions.input_ids[i] + eos_user_tokens.input_ids inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.exec_device)) + torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c36fd1de84c1..0692579cd2a0 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -111,7 +111,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) - self.word_embedding = self.llm.model.get_input_embeddings().to(self.exec_device) + self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): batch_size = len(question) From 62357033217719ab57f2a7d2621810e72ed805fd Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 14:46:12 -0700 Subject: [PATCH 426/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 0692579cd2a0..d04c42dd3951 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -16,7 +16,6 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): - assert torch.cuda.is_available(), "GPU needed to run LLMs efficiently!" avail_gpus = torch.cuda.device_count() kwargs = { "revision": "main", From 1f5c607ad9725c0353868554e18cacbbf126937b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 17:45:35 -0700 Subject: [PATCH 427/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 31 ++++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 07232e0ebfd7..db359829e816 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -95,7 +95,6 @@ def __init__( ) self.llm_generator = get_peft_model(self.llm_generator, config) self.llm_device = self.llm_to_use.llm_device - self.exec_device = self.llm_to_use.exec_device self.tokenizer = self.llm_to_use.tokenizer print('Finished loading LLAMA!') @@ -106,23 +105,23 @@ def __init__( num_layers=num_gnn_layers, heads=num_gnn_heads, norm='batch_norm', - ).to(self.exec_device) + ).to(self.llm_device) # For the MLP Projection mlp_hidden_dim = gnn_out_channels self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), nn.Linear(mlp_hidden_dim, mlp_out_dim), - ).to(self.exec_device) + ).to(self.llm_device) self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, node_feat, edge_index, edge_attr, batch): - x = node_feat.to(self.exec_device) - edge_index = edge_index.long().to(self.exec_device) - edge_attr = edge_attr.to(self.exec_device) + x = node_feat.to(self.llm_device) + edge_index = edge_index.long().to(self.llm_device) + edge_attr = edge_attr.to(self.llm_device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) - batch = batch.to(self.exec_device) + batch = batch.to(self.llm_device) # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds @@ -187,7 +186,7 @@ def forward( if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.exec_device) for i in to_cat], + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -201,8 +200,8 @@ def forward( for i in range(batch_size): pad_length = max_length - batch_inputs_embeds[i].shape[0] batch_inputs_embeds[i] = torch.cat([ - pad_embeds.repeat(pad_length, 1).to(self.exec_device), - batch_inputs_embeds[i].to(self.exec_device) + pad_embeds.repeat(pad_length, 1).to(self.llm_device), + batch_inputs_embeds[i].to(self.llm_device) ]) batch_attention_mask[i] = [0 ] * pad_length + batch_attention_mask[i] @@ -210,11 +209,11 @@ def forward( i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.exec_device) + dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to( - self.exec_device) + self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( - self.exec_device) + self.llm_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator( @@ -281,7 +280,7 @@ def inference( if num_nodes_per_graph[i] != 0: to_cat.append(graph_embeds[i].unsqueeze(0)) to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.exec_device) for i in to_cat], + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -296,9 +295,9 @@ def inference( ] * pad_length + batch_attention_mask[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.exec_device) + dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to( - self.exec_device) + self.llm_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( From b87843968eb8876237054facd47b80104e002340 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 00:46:44 +0000 Subject: [PATCH 428/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index db359829e816..1752e772b7ad 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -210,8 +210,7 @@ def forward( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) @@ -296,8 +295,7 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( From 7878f019c21d4e90eb9f1f1b4491f5eeb51a11b3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Wed, 22 May 2024 17:53:32 -0700 Subject: [PATCH 429/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index d04c42dd3951..22d8f11e62a1 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -38,9 +38,9 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict - kwargs["device_map"] = "auto" kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True + kwargs["device_map"] = "auto" return kwargs, cpu_offload @@ -95,14 +95,6 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if cpu_offload: self.llm_device = torch.device("cpu") - if torch.cuda.is_available(): - import accelerate - self.exec_device = torch.device("cuda:0") - self.llm = accelerate.cpu_offload( - self.llm, execution_device=self.exec_device, - offload_buffers=True) - else: - self.exec_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() else: From 920907b820a75fdc840359404740beb891203318 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 09:11:02 -0700 Subject: [PATCH 430/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 22d8f11e62a1..95b1ef73a68d 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -38,9 +38,9 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict - kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True kwargs["device_map"] = "auto" + kwargs["torch_dtype"] = autocast_dtype return kwargs, cpu_offload From 86734909ad5808b153daa5d7b3221042e2ddb9a9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 09:13:21 -0700 Subject: [PATCH 431/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 95b1ef73a68d..22d8f11e62a1 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -38,9 +38,9 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict + kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True kwargs["device_map"] = "auto" - kwargs["torch_dtype"] = autocast_dtype return kwargs, cpu_offload From 5259f2e46bc7ab47055b13f4e69f15b9732f02b3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:19:36 -0700 Subject: [PATCH 432/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 22d8f11e62a1..9cb2f3e06d57 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -2,6 +2,7 @@ import torch import torch.nn as nn +from torch.distributed import fsdp as FSDP BOS = '[INST]' EOS_USER = '[/INST]' @@ -97,6 +98,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() + self.llm = FSDP.FullyShardedDataParallel(self.llm, cpu_offload=FSDP.CPUOffload()) else: self.llm_device = self.llm.device self.exec_device = self.llm_device From c096a3c1459f3e957ddce6315bc89c7cdb946587 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:21:03 +0000 Subject: [PATCH 433/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 9cb2f3e06d57..c287b4f1c8ca 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -98,7 +98,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() - self.llm = FSDP.FullyShardedDataParallel(self.llm, cpu_offload=FSDP.CPUOffload()) + self.llm = FSDP.FullyShardedDataParallel( + self.llm, cpu_offload=FSDP.CPUOffload()) else: self.llm_device = self.llm.device self.exec_device = self.llm_device From 2adf084dd39cfb6634b0f08744c0a69f3d8100bf Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:22:01 -0700 Subject: [PATCH 434/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 9cb2f3e06d57..9e9379fe09d9 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -3,6 +3,7 @@ import torch import torch.nn as nn from torch.distributed import fsdp as FSDP +from torch import distributed as dist BOS = '[INST]' EOS_USER = '[/INST]' @@ -98,6 +99,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() + dist.init_process_group() self.llm = FSDP.FullyShardedDataParallel(self.llm, cpu_offload=FSDP.CPUOffload()) else: self.llm_device = self.llm.device From 18fe16903a3c5f2d3433607091281d08456f5101 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:24:16 +0000 Subject: [PATCH 435/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 7fb5d83b82d0..dc0eaf67a977 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -2,8 +2,8 @@ import torch import torch.nn as nn -from torch.distributed import fsdp as FSDP from torch import distributed as dist +from torch.distributed import fsdp as FSDP BOS = '[INST]' EOS_USER = '[/INST]' From cd0e37ed59792a065d5c6bd98df93b65c3c0f63a Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:30:42 -0700 Subject: [PATCH 436/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 7fb5d83b82d0..24b7cc1cfcdf 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -92,6 +92,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, use_fast=False) self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side + import inspect + print("inspect.getargspec(AutoModelForCausalLM.from_pretrained)=", inspect.getargspec(AutoModelForCausalLM.from_pretrained)) self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, **kwargs) @@ -99,9 +101,9 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() - dist.init_process_group() - self.llm = FSDP.FullyShardedDataParallel( - self.llm, cpu_offload=FSDP.CPUOffload()) + # dist.init_process_group() + # self.llm = FSDP.FullyShardedDataParallel( + # self.llm, cpu_offload=FSDP.CPUOffload()) else: self.llm_device = self.llm.device self.exec_device = self.llm_device From b4a8fa64d6427262997007324b0e304a00adacb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:31:50 +0000 Subject: [PATCH 437/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 70057b819e5d..c70a45e98c31 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -93,7 +93,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side import inspect - print("inspect.getargspec(AutoModelForCausalLM.from_pretrained)=", inspect.getargspec(AutoModelForCausalLM.from_pretrained)) + print("inspect.getargspec(AutoModelForCausalLM.from_pretrained)=", + inspect.getargspec(AutoModelForCausalLM.from_pretrained)) self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, **kwargs) From c0b124c65b9fb7ff64632d18ff6ff7aa56a685ac Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:40:54 -0700 Subject: [PATCH 438/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 1752e772b7ad..5eed5459a6f2 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -213,7 +213,7 @@ def forward( attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - + print("before LLM mem profile=", torch.cuda.mem_get_info()) with self.llm_to_use.autocast_context: outputs = self.llm_generator( inputs_embeds=inputs_embeds, From 510e5aee67048b5201992141a7d24715b7d7e5cc Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:47:41 -0700 Subject: [PATCH 439/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c70a45e98c31..73e9be5ab9ec 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -40,9 +40,10 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict - kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True kwargs["device_map"] = "auto" + kwargs["torch_dtype"] = autocast_dtype + return kwargs, cpu_offload @@ -92,9 +93,6 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, use_fast=False) self.tokenizer.pad_token_id = pad_token_id self.tokenizer.padding_side = padding_side - import inspect - print("inspect.getargspec(AutoModelForCausalLM.from_pretrained)=", - inspect.getargspec(AutoModelForCausalLM.from_pretrained)) self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, **kwargs) From 78415c5b42619cea09225abf846a32e754bd00e4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:48:48 +0000 Subject: [PATCH 440/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 73e9be5ab9ec..5e4f3c95d5af 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -44,7 +44,6 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["device_map"] = "auto" kwargs["torch_dtype"] = autocast_dtype - return kwargs, cpu_offload From e2b88e2e362a4cd4e38019300d1ce90a7d701d0c Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:52:37 -0700 Subject: [PATCH 441/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 73e9be5ab9ec..16d8daaf4671 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -42,8 +42,6 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["max_memory"] = max_mem_dict kwargs["low_cpu_mem_usage"] = True kwargs["device_map"] = "auto" - kwargs["torch_dtype"] = autocast_dtype - return kwargs, cpu_offload @@ -95,20 +93,16 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.tokenizer.padding_side = padding_side self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, **kwargs) - + self.word_embedding = self.llm.model.get_input_embeddings() if cpu_offload: self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() - # dist.init_process_group() - # self.llm = FSDP.FullyShardedDataParallel( - # self.llm, cpu_offload=FSDP.CPUOffload()) + accelerate.cpu_offload(self.word_embedding, "cuda") else: self.llm_device = self.llm.device - self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) - self.word_embedding = self.llm.model.get_input_embeddings() def encode_inputs(self, question, additional_context=None): batch_size = len(question) From 34d47da20eab2a1e5443cc4d8b84447b227e72d0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:54:49 +0000 Subject: [PATCH 442/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 11507567a39b..5c1a695352e3 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,7 +100,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, from contextlib import nullcontext self.autocast_context = nullcontext() import accelerate - self.word_embedding = accelerate.cpu_offload(self.word_embedding, "cuda") + self.word_embedding = accelerate.cpu_offload( + self.word_embedding, "cuda") self.llm = accelerate.cpu_offload(self.llm, "cuda") else: self.llm_device = self.llm.device From a8a01c7a3d7c80e0491f377ab048bc187d7a6f15 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:55:41 -0700 Subject: [PATCH 443/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 11507567a39b..9cb103bdfe47 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,8 +100,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, from contextlib import nullcontext self.autocast_context = nullcontext() import accelerate - self.word_embedding = accelerate.cpu_offload(self.word_embedding, "cuda") - self.llm = accelerate.cpu_offload(self.llm, "cuda") + self.llm = accelerate.cpu_offload(self.llm, execution_device="cuda") + self.word_embedding = accelerate.cpu_offload(self.word_embedding, execution_device="cuda") else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From ea817f63a691d4151456c8e58904316c8f7483ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 21:57:06 +0000 Subject: [PATCH 444/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 9cb103bdfe47..da0c0d7c380e 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -100,8 +100,10 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, from contextlib import nullcontext self.autocast_context = nullcontext() import accelerate - self.llm = accelerate.cpu_offload(self.llm, execution_device="cuda") - self.word_embedding = accelerate.cpu_offload(self.word_embedding, execution_device="cuda") + self.llm = accelerate.cpu_offload(self.llm, + execution_device="cuda") + self.word_embedding = accelerate.cpu_offload( + self.word_embedding, execution_device="cuda") else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From 78391523139368a587bddf7dd42d8008dd495941 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:58:05 -0700 Subject: [PATCH 445/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 9cb103bdfe47..f66b3531dd67 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -42,7 +42,7 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["max_memory"] = max_mem_dict kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True - kwargs["device_map"] = "auto" + kwargs["device_map"] = "auto" return kwargs, cpu_offload From 97e7b43ca2c922727a5d0922da575e8f6c5c0e5f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 14:59:53 -0700 Subject: [PATCH 446/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 91a547550fc4..e0054726f395 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -99,11 +99,12 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() - import accelerate - self.llm = accelerate.cpu_offload(self.llm, - execution_device="cuda") - self.word_embedding = accelerate.cpu_offload( - self.word_embedding, execution_device="cuda") + if torch.cuda.is_available(): + import accelerate + self.llm = accelerate.cpu_offload(self.llm, + execution_device="cuda") + self.word_embedding = accelerate.cpu_offload( + self.word_embedding, execution_device="cuda") else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From d0820822f80eecc63c94daa96af98cd4e9d56f81 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:01:03 -0700 Subject: [PATCH 447/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index e0054726f395..0a12c5db2b3d 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -103,8 +103,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, import accelerate self.llm = accelerate.cpu_offload(self.llm, execution_device="cuda") - self.word_embedding = accelerate.cpu_offload( - self.word_embedding, execution_device="cuda") + # self.word_embedding = accelerate.cpu_offload( + # self.word_embedding, execution_device="cuda") else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From bd6c40d22b3458f19e8c9cf3daf2a347cdcc1bb6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:06:07 -0700 Subject: [PATCH 448/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 12 ++++++------ torch_geometric/nn/text/llm.py | 8 ++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 5eed5459a6f2..accfe31115bb 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -95,6 +95,7 @@ def __init__( ) self.llm_generator = get_peft_model(self.llm_generator, config) self.llm_device = self.llm_to_use.llm_device + self.exec_device = self.llm_to_use.exec_device self.tokenizer = self.llm_to_use.tokenizer print('Finished loading LLAMA!') @@ -209,11 +210,10 @@ def forward( i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + dim=0).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) - print("before LLM mem profile=", torch.cuda.mem_get_info()) + self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator( inputs_embeds=inputs_embeds, @@ -294,8 +294,8 @@ def inference( ] * pad_length + batch_attention_mask[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + dim=0).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 0a12c5db2b3d..cf89c6827b32 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -101,12 +101,16 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.autocast_context = nullcontext() if torch.cuda.is_available(): import accelerate + self.exec_device = "cuda" self.llm = accelerate.cpu_offload(self.llm, - execution_device="cuda") + execution_device=self.exec_device) # self.word_embedding = accelerate.cpu_offload( - # self.word_embedding, execution_device="cuda") + # self.word_embedding, execution_device=self.exec_device) + else: + self.exec_device = "cpu" else: self.llm_device = self.llm.device + self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) From 774be43c9a58eff9f6589065823c5861ff7e7396 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 22:07:38 +0000 Subject: [PATCH 449/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 6 ++++-- torch_geometric/nn/text/llm.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index accfe31115bb..70cd259ea004 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -211,7 +211,8 @@ def forward( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.exec_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.exec_device) with self.llm_to_use.autocast_context: @@ -295,7 +296,8 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) + attention_mask = torch.tensor(batch_attention_mask).to( + self.exec_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index cf89c6827b32..c8f0ffa2f53c 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -102,8 +102,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, if torch.cuda.is_available(): import accelerate self.exec_device = "cuda" - self.llm = accelerate.cpu_offload(self.llm, - execution_device=self.exec_device) + self.llm = accelerate.cpu_offload( + self.llm, execution_device=self.exec_device) # self.word_embedding = accelerate.cpu_offload( # self.word_embedding, execution_device=self.exec_device) else: From 6b6c4ffd3d13c658df14de6531388342664f6145 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:08:56 -0700 Subject: [PATCH 450/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index accfe31115bb..b8ee7697b3d2 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -214,6 +214,7 @@ def forward( attention_mask = torch.tensor(batch_attention_mask).to(self.exec_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.exec_device) + print("before LLM mem profile=", torch.cuda.mem_get_info()) with self.llm_to_use.autocast_context: outputs = self.llm_generator( inputs_embeds=inputs_embeds, From b72096a7ef1f5067fa860ab4b08f1029d20d89bb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:12:06 -0700 Subject: [PATCH 451/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c8f0ffa2f53c..c033cf561cb1 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -42,7 +42,9 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["max_memory"] = max_mem_dict kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True - kwargs["device_map"] = "auto" + else: + kwargs["load_in_8bit"] = True + kwargs["device_map"] = "auto" return kwargs, cpu_offload @@ -99,18 +101,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm_device = torch.device("cpu") from contextlib import nullcontext self.autocast_context = nullcontext() - if torch.cuda.is_available(): - import accelerate - self.exec_device = "cuda" - self.llm = accelerate.cpu_offload( - self.llm, execution_device=self.exec_device) - # self.word_embedding = accelerate.cpu_offload( - # self.word_embedding, execution_device=self.exec_device) - else: - self.exec_device = "cpu" else: self.llm_device = self.llm.device - self.exec_device = self.llm_device self.autocast_context = torch.cuda.amp.autocast( dtype=self.llm_dtype) From 5dbb2cb55a6507316ef07f3104ca08254801e202 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:14:15 -0700 Subject: [PATCH 452/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index c0cd9730dfbd..5eed5459a6f2 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -95,7 +95,6 @@ def __init__( ) self.llm_generator = get_peft_model(self.llm_generator, config) self.llm_device = self.llm_to_use.llm_device - self.exec_device = self.llm_to_use.exec_device self.tokenizer = self.llm_to_use.tokenizer print('Finished loading LLAMA!') @@ -210,11 +209,10 @@ def forward( i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.exec_device) + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( - self.exec_device) + self.llm_device) print("before LLM mem profile=", torch.cuda.mem_get_info()) with self.llm_to_use.autocast_context: outputs = self.llm_generator( @@ -296,9 +294,8 @@ def inference( ] * pad_length + batch_attention_mask[i] inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.exec_device) - attention_mask = torch.tensor(batch_attention_mask).to( - self.exec_device) + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( From 45e33fd6c75e4c58ed984830932aa92c3a5cecea Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:16:17 -0700 Subject: [PATCH 453/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index c033cf561cb1..464ca6cd0672 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -43,7 +43,7 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True else: - kwargs["load_in_8bit"] = True + kwargs["load_in_4bit"] = True kwargs["device_map"] = "auto" return kwargs, cpu_offload From ddd671e3c49a9ad99d787358fdbd649f203d8142 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:26:04 -0700 Subject: [PATCH 454/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 464ca6cd0672..f8f26a155ddf 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -35,18 +35,21 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): # this is to minimize the need for interGPU communications if mem_total >= mem_needed: break - cpu_offload = mem_total < mem_needed - if not cpu_offload: + + # If not enough VRAM, we use pure CPU since huggingface automatic + # cpu offloading only works for inference. + # See: https://discuss.huggingface.co/t/device-map-auto/49610/2 + pure_cpu = mem_total < mem_needed + if not pure_cpu: for i in range(gpus_2_use_4_llm): max_mem_dict[i] = str(avail_mem_dict[i]) + "GiB" kwargs["max_memory"] = max_mem_dict - kwargs["torch_dtype"] = autocast_dtype kwargs["low_cpu_mem_usage"] = True - else: - kwargs["load_in_4bit"] = True - kwargs["device_map"] = "auto" + kwargs["device_map"] = "auto" + kwargs["torch_dtype"] = autocast_dtype + - return kwargs, cpu_offload + return kwargs, pure_cpu class LLM(nn.Module): @@ -88,7 +91,7 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.mem_needed = 85 * num_params / 7 self.llm_dtype = dtype print('Loading ' + str(self.printable_llm_name)) - kwargs, cpu_offload = get_llm_kwargs(self.mem_needed, self.llm_dtype) + kwargs, pure_cpu = get_llm_kwargs(self.mem_needed, self.llm_dtype) print("Setting up " + self.printable_llm_name + " w/ kwargs =", kwargs) self.tokenizer = AutoTokenizer.from_pretrained(self.huggingface_str, use_fast=False) @@ -97,10 +100,9 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.llm = AutoModelForCausalLM.from_pretrained( self.huggingface_str, **kwargs) self.word_embedding = self.llm.model.get_input_embeddings() - if cpu_offload: + if pure_cpu: self.llm_device = torch.device("cpu") - from contextlib import nullcontext - self.autocast_context = nullcontext() + self.autocast_context = torch.amp.autocast(device_type="cpu", dtype=self.llm_dtype) else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From 15591db42946d4f0ac9366007cc87481d76c7788 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 22:27:06 +0000 Subject: [PATCH 455/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index f8f26a155ddf..de0df6ee55f4 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -48,7 +48,6 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["device_map"] = "auto" kwargs["torch_dtype"] = autocast_dtype - return kwargs, pure_cpu @@ -102,7 +101,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.word_embedding = self.llm.model.get_input_embeddings() if pure_cpu: self.llm_device = torch.device("cpu") - self.autocast_context = torch.amp.autocast(device_type="cpu", dtype=self.llm_dtype) + self.autocast_context = torch.amp.autocast(device_type="cpu", + dtype=self.llm_dtype) else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From d31f5b90c323da8d2d976276a0fc8b61beea2d84 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:29:09 -0700 Subject: [PATCH 456/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/models/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 5eed5459a6f2..7dbfe98a19e3 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -213,7 +213,6 @@ def forward( attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) label_input_ids = torch.tensor(batch_label_input_ids).to( self.llm_device) - print("before LLM mem profile=", torch.cuda.mem_get_info()) with self.llm_to_use.autocast_context: outputs = self.llm_generator( inputs_embeds=inputs_embeds, From 6e0151f087dd6ff7da03915971a232d2fce22c61 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 23 May 2024 15:35:52 -0700 Subject: [PATCH 457/752] accomadating smaller gpus w/ cpu offloading --- torch_geometric/nn/text/llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index de0df6ee55f4..29579b582192 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -46,7 +46,7 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): kwargs["max_memory"] = max_mem_dict kwargs["low_cpu_mem_usage"] = True kwargs["device_map"] = "auto" - kwargs["torch_dtype"] = autocast_dtype + kwargs["torch_dtype"] = autocast_dtype return kwargs, pure_cpu @@ -101,8 +101,8 @@ def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, self.word_embedding = self.llm.model.get_input_embeddings() if pure_cpu: self.llm_device = torch.device("cpu") - self.autocast_context = torch.amp.autocast(device_type="cpu", - dtype=self.llm_dtype) + from contextlib import nullcontext + self.autocast_context = nullcontext() else: self.llm_device = self.llm.device self.autocast_context = torch.cuda.amp.autocast( From 60f5f540514ef394e02ddba8cdae5cafd3cce071 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 10:37:08 -0700 Subject: [PATCH 458/752] minor cleanup --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f8aa6399a25c..0bf996218657 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -228,8 +228,8 @@ def adjust_learning_rate(param_group, LR, epoch): model_save_name + "_best_val_loss_ckpt.pt") model.eval() eval_output = [] - progress_bar_test = tqdm(range(len(test_loader))) print("Final Evaluation...") + progress_bar_test = tqdm(range(len(test_loader))) for step, batch in enumerate(test_loader): with torch.no_grad(): output = inference_step(model, batch, model_save_name) From 789b16eb9be3ae2e499dc2c25c7a98bb27e8a097 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 11:03:02 -0700 Subject: [PATCH 459/752] minor cleanup --- examples/llm_plus_gnn/g_retriever.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0bf996218657..dbcd7bf76158 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -117,7 +117,7 @@ def inference_step(model, batch, model_save_name): def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, - eval_batch_size, lr, model=None, dataset=None, checkpointing=False): + eval_batch_size, lr, model=None, dataset=None, checkpointing=False, tiny_llama=False): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -150,8 +150,13 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: - model = GRetriever(gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers) + if tiny_llama: + model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers, mlp_out_dim=2048) + else: + model = GRetriever(gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers) if num_gnn_layers is not None: model_save_name = "gnn_llm" else: @@ -389,6 +394,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "--checkpointing", action="store_true", help="Use this flag to checkpoint each time a \ new best val loss is achieved") + parser.add_argument( + "--tiny_llama", action="store_true", + help="This example uses LLAMA2 (7B) by default. \ + This flag will run the example with TinyLLAMA (1B).") args = parser.parse_args() # check if saved model @@ -405,7 +414,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, prep_time, dataset, gnn_llm_eval_outs = train( since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, args.eval_batch_size, args.lr, - checkpointing=args.checkpointing) + checkpointing=args.checkpointing, tiny_llama=args.tiny_llama) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() From 1fe760022296866c5d8b248dd8c3186e1b978661 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 18:04:56 +0000 Subject: [PATCH 460/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index dbcd7bf76158..13383761295b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -117,7 +117,8 @@ def inference_step(model, batch, model_save_name): def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, - eval_batch_size, lr, model=None, dataset=None, checkpointing=False, tiny_llama=False): + eval_batch_size, lr, model=None, dataset=None, checkpointing=False, + tiny_llama=False): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -152,7 +153,7 @@ def adjust_learning_rate(param_group, LR, epoch): if model is None: if tiny_llama: model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - gnn_hidden_channels=hidden_channels, + gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers, mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, From 7a61e0d207a03e3c46830e2ab90857fde83d1052 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 11:07:02 -0700 Subject: [PATCH 461/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index dbcd7bf76158..7ac92f0dd9d3 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -152,8 +152,9 @@ def adjust_learning_rate(param_group, LR, epoch): if model is None: if tiny_llama: model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers, mlp_out_dim=2048) + gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers, + mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From 67a4486b22ae46d32a8941cb011547627061e3f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 18:09:03 +0000 Subject: [PATCH 462/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index dad88a8230c1..13383761295b 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -154,8 +154,7 @@ def adjust_learning_rate(param_group, LR, epoch): if tiny_llama: model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers, - mlp_out_dim=2048) + num_gnn_layers=num_gnn_layers, mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From bdc506c5cfb9cda8e05bc64f9f4c086b3a772626 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 11:13:10 -0700 Subject: [PATCH 463/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index dad88a8230c1..04b3b1f56f59 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -154,8 +154,8 @@ def adjust_learning_rate(param_group, LR, epoch): if tiny_llama: model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers, - mlp_out_dim=2048) + num_gnn_layers=num_gnn_layers,) + #mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From 2cbd80b450da53daaa9079f77c393e40c070d108 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 18:14:47 +0000 Subject: [PATCH 464/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 04b3b1f56f59..c4f8af4a8b4a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -152,10 +152,12 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: if tiny_llama: - model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers,) - #mlp_out_dim=2048) + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers, + ) + #mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From 157c910ed7dc0dfbd04dc24ceab21d930f6b18de Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:38:31 -0700 Subject: [PATCH 465/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 04b3b1f56f59..681beadd4c06 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -153,9 +153,9 @@ def adjust_learning_rate(param_group, LR, epoch): if model is None: if tiny_llama: model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers,) - #mlp_out_dim=2048) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From ad12cbb15cb76dfa7a11359fe9942303ea485c47 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:39:43 -0700 Subject: [PATCH 466/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 41a0bf719bb6..9d77829b86cf 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -152,6 +152,7 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: if tiny_llama: + print("tiny_llama...") model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", num_llm_params=1, # 1 Billion gnn_hidden_channels=hidden_channels, From 7ba9b2fd9e6966db428ab27143c6db0367f77de6 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:40:34 -0700 Subject: [PATCH 467/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9d77829b86cf..41a0bf719bb6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -152,7 +152,6 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: if tiny_llama: - print("tiny_llama...") model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", num_llm_params=1, # 1 Billion gnn_hidden_channels=hidden_channels, From 108f362f2f81108ff43279870a3a3f4507697823 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 20:42:07 +0000 Subject: [PATCH 468/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 41a0bf719bb6..9ab1811631e4 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -152,11 +152,12 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: if tiny_llama: - model = GRetriever(llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - num_llm_params=1, # 1 Billion - gnn_hidden_channels=hidden_channels, - num_gnn_layers=num_gnn_layers, - ) + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + gnn_hidden_channels=hidden_channels, + num_gnn_layers=num_gnn_layers, + ) else: model = GRetriever(gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) From 3084d34156d756f12b1697b1d78f72a39546d382 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:42:15 -0700 Subject: [PATCH 469/752] adding tiny-llama --- torch_geometric/nn/models/g_retriever.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 7dbfe98a19e3..94f7b41ecab1 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -63,11 +63,11 @@ def __init__( ) -> None: super().__init__() if 'llama' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2', llm_dtype) + self.llm_to_use = LLM('llama2', llm_dtype, num_params=num_llm_params) elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma', llm_dtype) + self.llm_to_use = LLM('gemma', llm_dtype, num_params=num_llm_params) else: - self.llm_to_use = LLM(llm_to_use, llm_dtype) + self.llm_to_use = LLM(llm_to_use, llm_dtype, num_params=num_llm_params) self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: From e4008bbf1c7c50d637e60a9d6459d7c9008dc17a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 20:44:20 +0000 Subject: [PATCH 470/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 94f7b41ecab1..ed55db1dbab0 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -63,11 +63,14 @@ def __init__( ) -> None: super().__init__() if 'llama' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2', llm_dtype, num_params=num_llm_params) + self.llm_to_use = LLM('llama2', llm_dtype, + num_params=num_llm_params) elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma', llm_dtype, num_params=num_llm_params) + self.llm_to_use = LLM('gemma', llm_dtype, + num_params=num_llm_params) else: - self.llm_to_use = LLM(llm_to_use, llm_dtype, num_params=num_llm_params) + self.llm_to_use = LLM(llm_to_use, llm_dtype, + num_params=num_llm_params) self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: From 56a3912cb7b5dba19771f0aa857086a6bd29c911 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:45:44 -0700 Subject: [PATCH 471/752] adding tiny-llama --- torch_geometric/nn/models/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 94f7b41ecab1..d6a89163bce1 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -67,6 +67,7 @@ def __init__( elif 'gemma' in llm_to_use.lower(): self.llm_to_use = LLM('gemma', llm_dtype, num_params=num_llm_params) else: + print("inside else for picking other model...") self.llm_to_use = LLM(llm_to_use, llm_dtype, num_params=num_llm_params) self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype From e54cc364cf7a7857fb9f9941abb5fd3dc95aafc3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:48:24 -0700 Subject: [PATCH 472/752] adding tiny-llama --- torch_geometric/nn/models/g_retriever.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 34d9f3738516..41f4cd4ddd6f 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -48,7 +48,7 @@ class GRetriever(nn.Module): """ def __init__( self, - llm_to_use='llama2', + llm_to_use='llama2-7b', llm_use_lora: bool = False, llm_dtype=torch.bfloat16, num_llm_params: int = 7, @@ -62,8 +62,8 @@ def __init__( mlp_out_dim: int = 4096, ) -> None: super().__init__() - if 'llama' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2', llm_dtype, + if 'llama2-7b' in llm_to_use.lower(): + self.llm_to_use = LLM('llama2-7b', llm_dtype, num_params=num_llm_params) elif 'gemma' in llm_to_use.lower(): self.llm_to_use = LLM('gemma', llm_dtype, @@ -100,8 +100,6 @@ def __init__( self.llm_generator = get_peft_model(self.llm_generator, config) self.llm_device = self.llm_to_use.llm_device self.tokenizer = self.llm_to_use.tokenizer - print('Finished loading LLAMA!') - self.graph_encoder = gnn_to_use( in_channels=gnn_in_channels, out_channels=gnn_out_channels, From 44581d39d6c4a5c6c07a2352d7fdca914d8a8534 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:48:53 -0700 Subject: [PATCH 473/752] adding tiny-llama --- torch_geometric/nn/models/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 41f4cd4ddd6f..e9c04838f5a1 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -69,7 +69,6 @@ def __init__( self.llm_to_use = LLM('gemma', llm_dtype, num_params=num_llm_params) else: - print("inside else for picking other model...") self.llm_to_use = LLM(llm_to_use, llm_dtype, num_params=num_llm_params) self.llm_generator = self.llm_to_use.llm From 73515644c89935edbef2f31a62e334f189cc6d35 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:49:54 -0700 Subject: [PATCH 474/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9ab1811631e4..3c1108c07f99 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -157,6 +157,7 @@ def adjust_learning_rate(param_group, LR, epoch): num_llm_params=1, # 1 Billion gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers, + mlp_out_dim=2048, ) else: model = GRetriever(gnn_hidden_channels=hidden_channels, From eda47efeaf10989e5180300ec4d32264ed39f0b2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 13:55:53 -0700 Subject: [PATCH 475/752] adding tiny-llama --- torch_geometric/nn/text/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 29579b582192..2dcf228cd8a6 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -68,11 +68,11 @@ class LLM(nn.Module): automatically allocate the number of gpus needed, given the available GPU memory of your GPUs (default :obj:`7`) """ - def __init__(self, model_name: str = "llama2", dtype=torch.bfloat16, + def __init__(self, model_name: str = "llama2-7b", dtype=torch.bfloat16, num_params: int = 7): super().__init__() from transformers import AutoModelForCausalLM, AutoTokenizer - if model_name == "llama2": + if model_name == "llama2-7b": self.printable_llm_name = "LLAMA2" self.huggingface_str = llama2_str_name elif model_name == "gemma": From 43d2911dd7391622f0d82e3cce60fca1a9e9a144 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 28 May 2024 14:07:33 -0700 Subject: [PATCH 476/752] adding tiny-llama --- examples/llm_plus_gnn/g_retriever.py | 7 ++++--- torch_geometric/nn/text/llm.py | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3c1108c07f99..13b58fcc2e2e 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -1,9 +1,10 @@ -"""This example implements G-retriever using PyG. +""" +This example implements G-retriever using PyG. Original Paper: https://arxiv.org/abs/2402.07630 “G-Retriever significantly reduces hallucinations by 54% compared to the [LLM] baseline“. -requirements on top of basic PyG: -pip install datasets transformers pcst_fast sentencepiece tqdm pandas accelerate +Requirements on top of basic PyG: +`pip install datasets transformers pcst_fast sentencepiece tqdm accelerate`. """ import argparse import gc diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index 2dcf228cd8a6..f9201c33cd83 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -2,8 +2,6 @@ import torch import torch.nn as nn -from torch import distributed as dist -from torch.distributed import fsdp as FSDP BOS = '[INST]' EOS_USER = '[/INST]' From 46c389d88f376d9740b4a44652b40539d8a3685d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 21:08:37 +0000 Subject: [PATCH 477/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 13b58fcc2e2e..fa6e8048143a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -1,5 +1,4 @@ -""" -This example implements G-retriever using PyG. +"""This example implements G-retriever using PyG. Original Paper: https://arxiv.org/abs/2402.07630 “G-Retriever significantly reduces hallucinations by 54% compared to the [LLM] baseline“. From 07bb32a778a8f83c23c368bee36a1cf86b458b11 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 30 May 2024 11:00:46 -0700 Subject: [PATCH 478/752] GNN LLM Upgrades (#9374) closing https://github.com/pyg-team/pytorch_geometric/pull/9320 but saving the code associated improvements --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- examples/llm_plus_gnn/g_retriever.py | 88 ++++++++++++------- torch_geometric/datasets/web_qsp_dataset.py | 13 ++- torch_geometric/nn/text/llm.py | 67 ++++++++++---- .../nn/text/sentence_transformer_embedding.py | 51 ++++++++--- 4 files changed, 149 insertions(+), 70 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index fa6e8048143a..8842951b8a30 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -117,8 +117,8 @@ def inference_step(model, batch, model_save_name): def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, - eval_batch_size, lr, model=None, dataset=None, checkpointing=False, - tiny_llama=False): + eval_batch_size, lr, loss_fn, inference_fn, model=None, dataset=None, + checkpointing=False, tiny_llama=False): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -197,7 +197,7 @@ def adjust_learning_rate(param_group, LR, epoch): loader = tqdm(train_loader, desc=epoch_str) for step, batch in enumerate(loader): optimizer.zero_grad() - loss = get_loss(model, batch, model_save_name) + loss = loss_fn(model, batch, model_save_name) loss.backward() clip_grad_norm_(optimizer.param_groups[0]['params'], 0.1) @@ -219,7 +219,7 @@ def adjust_learning_rate(param_group, LR, epoch): model.eval() with torch.no_grad(): for step, batch in enumerate(val_loader): - loss = get_loss(model, batch, model_save_name) + loss = loss_fn(model, batch, model_save_name) val_loss += loss.item() val_loss = val_loss / len(val_loader) print(epoch_str + f", Val Loss: {val_loss}") @@ -242,7 +242,7 @@ def adjust_learning_rate(param_group, LR, epoch): progress_bar_test = tqdm(range(len(test_loader))) for step, batch in enumerate(test_loader): with torch.no_grad(): - output = inference_step(model, batch, model_save_name) + output = inference_fn(model, batch, model_save_name) output["label"] = batch.label eval_output.append(output) progress_bar_test.update(1) @@ -260,8 +260,10 @@ def adjust_learning_rate(param_group, LR, epoch): def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, - eval_batch_size): - print("First comparing against a pretrained LLAMA2 model...") + eval_batch_size, loss_fn, inference_fn, + skip_pretrained_LLM=False): + if not skip_pretrained_LLM: + print("First comparing against a pretrained LLM...") # Step 1: Define a single batch size test loader idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] @@ -290,30 +292,38 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] - # GNN+LLM only using 32 tokens to answer, give untrained LLM more - pure_llm_out = pure_llm.inference(batch.question, batch.desc, - max_out_tokens=256) + if skip_pretrained_LLM: + pure_llm_pred = None + pure_llm_hallucinates = False + else: + # GNN+LLM only using 32 tokens to answer, give untrained LLM more + pure_llm_out = pure_llm.inference(batch.question, batch.desc, + max_out_tokens=256) + pure_llm_pred = pure_llm_out['pred'][0] + pure_llm_hallucinates = detect_hallucinate( + pure_llm_pred, correct_answer) + untuned_llm_save_list += [(pure_llm_pred, + pure_llm_hallucinates)] + gnn_llm_pred = gnn_llm_preds[i] - pure_llm_pred = pure_llm_out['pred'][0] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) gnn_save_list += [(gnn_llm_pred, gnn_llm_hallucinates)] - pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, - correct_answer) - untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa # skipping when hallucination is hard to eval continue gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) pure_llm_hallucin_sum += int(pure_llm_hallucinates) - print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) - print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) - percent = 100.0 * round( - 1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) - print(f"GNN reduces pretrained LLM hallucinations by: ~{percent}%") - print("Note: hallucinations detected by regex hence the ~") - print("Now we see how the LLM compares when finetuned...") - print("Saving outputs of GNN+LLM and pretrained LLM...") + if not skip_pretrained_LLM: + print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) + print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) + percent = 100.0 * round( + 1 - (gnn_llm_hallucin_sum / pure_llm_hallucin_sum), 2) + print(f"GNN reduces pretrained LLM hallucinations by: ~{percent}%") + print("Note: hallucinations detected by regex hence the ~") + print("Now we see how the LLM compares when finetuned...") + print("Saving outputs of GNN+LLM and pretrained LLM...") save_dict = { "gnn_save_list": gnn_save_list, "untuned_llm_save_list": untuned_llm_save_list, @@ -333,18 +343,19 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, untuned_llm_hallucin_sum = pure_llm_hallucin_sum final_prnt_str = "" if path.exists("llm.pt") and path.exists("llm_eval_outs.pt"): - print("Existing finetuned LLAMA2 found.") + print("Existing finetuned LLM found.") print("Would you like to retrain?") user_input = str(input("(y/n):")).lower() retrain = user_input == "y" else: retrain = True if retrain: - print("Finetuning LLAMA2...") + print("Finetuning LLM...") since = time.time() _, _, pure_llm_eval_outputs = train(since, 1, None, None, batch_size, - eval_batch_size, lr, - model=pure_llm, dataset=dataset) + eval_batch_size, lr, loss_fn, + inference_fn, model=pure_llm, + dataset=dataset) e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: @@ -366,19 +377,26 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if pure_llm_hallucinates == "skip": continue trained_llm_hallucin_sum += int(pure_llm_hallucinates) + if skip_pretrained_LLM: + # we did not check the untrained LLM, so do not decide to demo + # based on this. + untuned_llm_hallucinates = True if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa final_prnt_str += "Prompt: '" + question + "'\n" final_prnt_str += "Label: '" + correct_answer + "'\n" - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa + if not skip_pretrained_LLM: + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" final_prnt_str += "\n" + "#" * 20 + "\n\n" - print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) + if not skip_pretrained_LLM: + print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) - percent = 100.0 * round( - 1 - (gnn_llm_hallucin_sum / untuned_llm_hallucin_sum), 2) - print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") + if not skip_pretrained_LLM: + percent = 100.0 * round( + 1 - (gnn_llm_hallucin_sum / untuned_llm_hallucin_sum), 2) + print(f"GNN reduces untuned LLM hallucinations by: ~{percent}%") tuned_percent = 100.0 * round( 1 - (gnn_llm_hallucin_sum / trained_llm_hallucin_sum), 2) print(f"GNN reduces tuned LLM hallucinations by: ~{tuned_percent}%") @@ -418,8 +436,9 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, since = time.time() prep_time, dataset, gnn_llm_eval_outs = train( since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, - args.batch_size, args.eval_batch_size, args.lr, - checkpointing=args.checkpointing, tiny_llama=args.tiny_llama) + args.batch_size, args.eval_batch_size, args.lr, get_loss, + inference_step, checkpointing=args.checkpointing, + tiny_llama=args.tiny_llama) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -431,4 +450,5 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, dataset = WebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, - args.batch_size, args.eval_batch_size) + args.batch_size, args.eval_batch_size, get_loss, + inference_step) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 27d6b459c493..376cacc89197 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -200,14 +200,13 @@ def download(self) -> None: def process(self) -> None: pretrained_repo = "sentence-transformers/all-roberta-large-v1" - self.model = SentenceTransformer(pretrained_repo) - self.model.to(self.device) + self.model = SentenceTransformer(pretrained_repo, device=self.device) self.model.eval() self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] # encode questions print("Encoding questions...") - q_embs = text2embedding(self.model, self.device, self.questions) + q_embs = text2embedding(self.model, self.questions, device=self.device) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] @@ -233,11 +232,11 @@ def process(self) -> None: edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") - x = text2embedding(self.model, self.device, - nodes.node_attr.tolist()) + x = text2embedding(self.model, nodes.node_attr.tolist(), + device=self.device) # encode edges - edge_attr = text2embedding(self.model, self.device, - edges.edge_attr.tolist()) + edge_attr = text2embedding(self.model, edges.edge_attr.tolist(), + device=self.device) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) question = f"Question: {data_i['question']}\nAnswer: " diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/text/llm.py index f9201c33cd83..619b0e78a4e8 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/text/llm.py @@ -49,6 +49,15 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): return kwargs, pure_cpu +def get_mem_needed_for_llm(num_params): + """This is a rough hueristic: + We found that LLAMA2 (7B) + GAT hits OOM + on a single 80GB GPU, but can fit on a single + GPU that is slightly larger GPU. + """ + return 85 * num_params / 7 + + class LLM(nn.Module): r"""This module wraps a HuggingFace Transformer based model. @@ -79,13 +88,7 @@ def __init__(self, model_name: str = "llama2-7b", dtype=torch.bfloat16, else: self.printable_llm_name = model_name self.huggingface_str = model_name - """ - This is a rough hueristic: - We found that LLAMA2 (7B) + GAT hits OOM - on a single 80GB GPU, but can fit on a single - GPU that is slightly larger GPU. - """ - self.mem_needed = 85 * num_params / 7 + self.mem_needed = get_mem_needed_for_llm(num_params) self.llm_dtype = dtype print('Loading ' + str(self.printable_llm_name)) kwargs, pure_cpu = get_llm_kwargs(self.mem_needed, self.llm_dtype) @@ -125,18 +128,31 @@ def encode_inputs(self, question, additional_context=None): return (batch_size, questions, additional_context, eos_user_tokens, bos_embeds, pad_embeds) - def forward(self, question: List[str], label: List[str], - additional_context: Optional[List[str]] = None): + def forward( + self, + question: List[str], + label: List[str], + additional_text_context: Optional[List[str]] = None, + rag_embeddings: Optional[List[torch.tensor]] = None, + ): r"""Forward pass. Args: question (List[str]): The questions/prompts. label (List[str]): The answers/labels. - additional_context (List[str], optional): Additional context to - give to the LLM, such as textified knowledge graphs. + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + rag_embeddings (List[torch.tensor], optional): Embedding tensors from RAG + docs, essentially just the embedded form of + `additional_text_context`. Only one of `additional_text_context` + or `rag_embeddings` should be used. """ + if rag_embeddings is not None and additional_context is not None: + print("Warning: Using both `rag_embeddings` and \ + `additional_context` inputs is a waste of compute & memory \ + as both provide the same information") batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.encode_inputs(question, additional_context) # noqa + bos_embeds, pad_embeds = self.encode_inputs(question, additional_text_context) # noqa # encode labels labels = self.tokenizer(label, add_special_tokens=False) # encode training specific special token @@ -159,6 +175,8 @@ def forward(self, question: List[str], label: List[str], inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.llm_device)) to_cat = [bos_embeds] + if rag_embeddings is not None and rag_embeddings[i] is not None: + to_cat.append(rag_embeddings[i]) to_cat.append(inputs_embeds) inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) @@ -196,19 +214,28 @@ def forward(self, question: List[str], label: List[str], @torch.no_grad() def inference(self, question: List[str], - additional_context: Optional[List[str]] = None, + additional_text_context: Optional[List[str]] = None, + rag_embeddings: Optional[List[torch.tensor]] = None, max_out_tokens: Optional[int] = max_new_tokens): r"""Inference. Args: question (List[str]): The questions/prompts. - additional_context (List[str], optional): Additional context to - give to the LLM, such as textified knowledge graphs. + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + rag_embeddings (torch.tensor, optional): Embedding tensor from RAG + docs, essentially just the embedded form of + `additional_text_context`. Only one of `additional_text_context` + or `rag_embeddings` should be used. max_out_tokens (int, optional): How many tokens for the LLM to generate. (default: {32}) """ + if rag_embeddings is not None and additional_context is not None: + print("Warning: Using both `rag_embeddings` and \ + `additional_context` inputs is a waste of compute & memory \ + as both provide the same information") batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.encode_inputs(question, additional_context) # noqa + bos_embeds, pad_embeds = self.encode_inputs(question, additional_text_context) # noqa batch_inputs_embeds = [] batch_attention_mask = [] for i in range(batch_size): @@ -222,7 +249,11 @@ def inference(self, question: List[str], inputs_embeds = self.word_embedding( torch.tensor(input_ids).to(self.llm_device)) - inputs_embeds = torch.cat([bos_embeds, inputs_embeds], dim=0) + to_cat = [bos_embeds] + if rag_embeddings is not None and rag_embeddings[i] is not None: + to_cat.append(rag_embeddings[i]) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat(to_cat, dim=0) batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.shape[0]) @@ -252,5 +283,5 @@ def inference(self, question: List[str], return { 'pred': pred, 'question': question, - 'desc': additional_context, + 'desc': additional_text_context, } diff --git a/torch_geometric/nn/text/sentence_transformer_embedding.py b/torch_geometric/nn/text/sentence_transformer_embedding.py index 180a8d1c55e8..0398afb2534e 100644 --- a/torch_geometric/nn/text/sentence_transformer_embedding.py +++ b/torch_geometric/nn/text/sentence_transformer_embedding.py @@ -1,16 +1,39 @@ +from contextlib import nullcontext from typing import List, Optional import torch import torch.nn.functional as F +from .llm import get_llm_kwargs, get_mem_needed_for_llm, pad_token_id + class SentenceTransformer(torch.nn.Module): - def __init__(self, pretrained_repo: str) -> None: + def __init__(self, pretrained_repo: str, + device: Optional[torch.device] = None, + autocast_dtype: Optional[torch.dtype] = None, + num_params: int = 7) -> None: super().__init__() print(f"inherit model weights from {pretrained_repo}") from transformers import AutoModel, AutoTokenizer - self.bert_model = AutoModel.from_pretrained(pretrained_repo) - self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo) + self.autocast_dtype = autocast_dtype + self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo, + use_fast=True) + self.tokenizer.pad_token_id = pad_token_id + if device is not None: + self.llm = AutoModel.from_pretrained(pretrained_repo).to(device) + self.llm_device = device + else: + if self.autocast_dtype is None: + self.autocast_dtype = torch.bfloat16 + self.mem_needed = get_mem_needed_for_llm(num_params) + kwargs, _ = get_llm_kwargs(self.mem_needed) + self.llm = AutoModel.from_pretrained(pretrained_repo, **kwargs) + self.llm_device = self.llm.device + if self.autocast_dtype is None: + self.autocast_context = nullcontext() + else: + self.autocast_context = torch.cuda.amp.autocast( + dtype=self.autocast_dtype) def mean_pooling(self, token_embeddings: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor: @@ -22,8 +45,8 @@ def mean_pooling(self, token_embeddings: torch.Tensor, def forward(self, input_ids: torch.Tensor, att_mask: torch.Tensor) -> torch.Tensor: - bert_out = self.bert_model(input_ids=input_ids, - attention_mask=att_mask) + with self.autocast_context: + bert_out = self.llm(input_ids=input_ids, attention_mask=att_mask) # First element of model_output contains all token embeddings token_embeddings = bert_out[0] @@ -32,17 +55,24 @@ def forward(self, input_ids: torch.Tensor, return sentence_embeddings -def text2embedding(model: SentenceTransformer, device: torch.device, - text: List[str], - batch_size: Optional[int] = 256) -> torch.Tensor: +def text2embedding( + model: SentenceTransformer, + text: List[str], + batch_size: Optional[int] = 256, + device: Optional[torch.device] = None, + truncate_long_strs=True, +) -> torch.Tensor: try: - encoding = model.tokenizer(text, padding=True, truncation=True, + encoding = model.tokenizer(text, padding=True, + truncation=truncate_long_strs, return_tensors="pt") data_len = encoding.input_ids.size(0) num_full_batches = data_len // batch_size all_embeddings_list = [] # Iterate through batches + if device is None: + device = model.llm_device with torch.no_grad(): left_ptr = 0 for i in range(num_full_batches): @@ -64,8 +94,7 @@ def text2embedding(model: SentenceTransformer, device: torch.device, # Concatenate the embeddings from all batches all_embeddings = torch.cat(all_embeddings_list, dim=0).cpu() except: # noqa - print( - "SBERT text embedding failed, returning torch.zeros((0, 1024))...") + print("text embedding failed, returning torch.zeros((0, 1024))...") return torch.zeros((0, 1024)) return all_embeddings From ca0ee15fc511ec31261a6229ed8e2c1a640cb384 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 30 May 2024 15:55:51 -0700 Subject: [PATCH 479/752] Align sent trans (#9376) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- examples/llm_plus_gnn/g_retriever.py | 7 +- torch_geometric/datasets/web_qsp_dataset.py | 14 ++- torch_geometric/nn/models/g_retriever.py | 4 +- torch_geometric/nn/nlp/__init__.py | 2 + torch_geometric/nn/{text => nlp}/llm.py | 19 ++-- .../nn/nlp/sentence_transformer.py | 15 ++- torch_geometric/nn/text/__init__.py | 10 -- .../nn/text/sentence_transformer_embedding.py | 100 ------------------ 8 files changed, 38 insertions(+), 133 deletions(-) rename torch_geometric/nn/{text => nlp}/llm.py (95%) delete mode 100644 torch_geometric/nn/text/__init__.py delete mode 100644 torch_geometric/nn/text/sentence_transformer_embedding.py diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 8842951b8a30..3040d4f49c03 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -21,7 +21,7 @@ from torch_geometric.datasets import WebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever -from torch_geometric.nn.text import LLM +from torch_geometric.nn.nlp import LLM def detect_hallucinate(pred, label): @@ -135,6 +135,7 @@ def adjust_learning_rate(param_group, LR, epoch): seed_everything(42) if dataset is None: dataset = WebQSPDataset() + gc.collect() idx_split = dataset.split_idxs # Step 1: Build Node Classification Dataset @@ -151,6 +152,7 @@ def adjust_learning_rate(param_group, LR, epoch): # Step 2: Build Model if model is None: + gc.collect() if tiny_llama: model = GRetriever( llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", @@ -296,7 +298,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_pred = None pure_llm_hallucinates = False else: - # GNN+LLM only using 32 tokens to answer, give untrained LLM more + # GNN+LLM only using 32 tokens to answer. + # Allow more output tokens for untrained LLM pure_llm_out = pure_llm.inference(batch.question, batch.desc, max_out_tokens=256) pure_llm_pred = pure_llm_out['pred'][0] diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 376cacc89197..5a13fcefd904 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -7,7 +7,7 @@ from tqdm import tqdm from torch_geometric.data import Data, InMemoryDataset -from torch_geometric.nn.text import SentenceTransformer, text2embedding +from torch_geometric.nn.nlp import SentenceTransformer try: from pandas import DataFrame @@ -199,14 +199,13 @@ def download(self) -> None: } def process(self) -> None: - pretrained_repo = "sentence-transformers/all-roberta-large-v1" - self.model = SentenceTransformer(pretrained_repo, device=self.device) + self.model = SentenceTransformer().to(self.device) self.model.eval() self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] # encode questions print("Encoding questions...") - q_embs = text2embedding(self.model, self.questions, device=self.device) + q_embs = self.model.encode(self.questions, batch_size=256) print("Encoding graphs...") for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] @@ -232,11 +231,10 @@ def process(self) -> None: edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") - x = text2embedding(self.model, nodes.node_attr.tolist(), - device=self.device) + x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) # encode edges - edge_attr = text2embedding(self.model, edges.edge_attr.tolist(), - device=self.device) + edge_attr = self.model.encode(edges.edge_attr.tolist(), + batch_size=256) edge_index = torch.LongTensor( [edges.src.tolist(), edges.dst.tolist()]) question = f"Question: {data_i['question']}\nAnswer: " diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index e9c04838f5a1..ce6d2c5c6759 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -4,10 +4,10 @@ import torch.nn as nn from torch_geometric.nn.models import GAT -from torch_geometric.nn.text import LLM -from torch_geometric.nn.text.llm import ( +from torch_geometric.nn.nlp.llm import ( EOS, IGNORE_INDEX, + LLM, max_new_tokens, max_txt_len, ) diff --git a/torch_geometric/nn/nlp/__init__.py b/torch_geometric/nn/nlp/__init__.py index 8b819f0a3747..c101a359e3f5 100644 --- a/torch_geometric/nn/nlp/__init__.py +++ b/torch_geometric/nn/nlp/__init__.py @@ -1,5 +1,7 @@ from .sentence_transformer import SentenceTransformer +from .llm import LLM __all__ = classes = [ 'SentenceTransformer', + 'LLM', ] diff --git a/torch_geometric/nn/text/llm.py b/torch_geometric/nn/nlp/llm.py similarity index 95% rename from torch_geometric/nn/text/llm.py rename to torch_geometric/nn/nlp/llm.py index 619b0e78a4e8..ed0ae09cfedc 100644 --- a/torch_geometric/nn/text/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -1,3 +1,4 @@ +import gc from typing import List, Optional import torch @@ -16,6 +17,8 @@ def get_llm_kwargs(mem_needed, autocast_dtype=torch.bfloat16): + torch.cuda.empty_cache() + gc.collect() avail_gpus = torch.cuda.device_count() kwargs = { "revision": "main", @@ -142,12 +145,12 @@ def forward( label (List[str]): The answers/labels. additional_text_context (List[str], optional): Additional context to give to the LLM, such as textified knowledge graphs. - rag_embeddings (List[torch.tensor], optional): Embedding tensors from RAG - docs, essentially just the embedded form of - `additional_text_context`. Only one of `additional_text_context` - or `rag_embeddings` should be used. + rag_embeddings (List[torch.tensor], optional): Embedding tensors + from RAG docs. Essentially just the embedded form of + `additional_text_context`. Either `additional_text_context` + or `rag_embeddings` should be used, not both. """ - if rag_embeddings is not None and additional_context is not None: + if rag_embeddings is not None and additional_text_context is not None: print("Warning: Using both `rag_embeddings` and \ `additional_context` inputs is a waste of compute & memory \ as both provide the same information") @@ -225,12 +228,12 @@ def inference(self, question: List[str], to give to the LLM, such as textified knowledge graphs. rag_embeddings (torch.tensor, optional): Embedding tensor from RAG docs, essentially just the embedded form of - `additional_text_context`. Only one of `additional_text_context` - or `rag_embeddings` should be used. + `additional_text_context`. Either `additional_text_context` + or `rag_embeddings` should be used, not both. max_out_tokens (int, optional): How many tokens for the LLM to generate. (default: {32}) """ - if rag_embeddings is not None and additional_context is not None: + if rag_embeddings is not None and additional_text_context is not None: print("Warning: Using both `rag_embeddings` and \ `additional_context` inputs is a waste of compute & memory \ as both provide the same information") diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index f82b77c2e6f1..59a74d080740 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -6,9 +6,17 @@ class SentenceTransformer(torch.nn.Module): - def __init__(self, model_name: str) -> None: - super().__init__() + r"""Embeds text as a vector using Huggingface transformers. + model_name (str): A string representing the huggingface model to use. + (default: :obj:`"sentence-transformers/all-roberta-large-v1"`) + """ + def __init__( + self, + model_name: Optional[ + str] = "sentence-transformers/all-roberta-large-v1", + ) -> None: + super().__init__() self.model_name = model_name from transformers import AutoModel, AutoTokenizer @@ -39,8 +47,9 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: + if len(text) == 0: + return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size - embs: List[Tensor] = [] for start in range(0, len(text), batch_size): token = self.tokenizer( diff --git a/torch_geometric/nn/text/__init__.py b/torch_geometric/nn/text/__init__.py deleted file mode 100644 index 7d2d18a3ba2f..000000000000 --- a/torch_geometric/nn/text/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -r"""Model package.""" - -from .llm import LLM -from .sentence_transformer_embedding import SentenceTransformer, text2embedding - -__all__ = classes = [ - 'LLM', - 'SentenceTransformer', - 'text2embedding', -] diff --git a/torch_geometric/nn/text/sentence_transformer_embedding.py b/torch_geometric/nn/text/sentence_transformer_embedding.py deleted file mode 100644 index 0398afb2534e..000000000000 --- a/torch_geometric/nn/text/sentence_transformer_embedding.py +++ /dev/null @@ -1,100 +0,0 @@ -from contextlib import nullcontext -from typing import List, Optional - -import torch -import torch.nn.functional as F - -from .llm import get_llm_kwargs, get_mem_needed_for_llm, pad_token_id - - -class SentenceTransformer(torch.nn.Module): - def __init__(self, pretrained_repo: str, - device: Optional[torch.device] = None, - autocast_dtype: Optional[torch.dtype] = None, - num_params: int = 7) -> None: - super().__init__() - print(f"inherit model weights from {pretrained_repo}") - from transformers import AutoModel, AutoTokenizer - self.autocast_dtype = autocast_dtype - self.tokenizer = AutoTokenizer.from_pretrained(pretrained_repo, - use_fast=True) - self.tokenizer.pad_token_id = pad_token_id - if device is not None: - self.llm = AutoModel.from_pretrained(pretrained_repo).to(device) - self.llm_device = device - else: - if self.autocast_dtype is None: - self.autocast_dtype = torch.bfloat16 - self.mem_needed = get_mem_needed_for_llm(num_params) - kwargs, _ = get_llm_kwargs(self.mem_needed) - self.llm = AutoModel.from_pretrained(pretrained_repo, **kwargs) - self.llm_device = self.llm.device - if self.autocast_dtype is None: - self.autocast_context = nullcontext() - else: - self.autocast_context = torch.cuda.amp.autocast( - dtype=self.autocast_dtype) - - def mean_pooling(self, token_embeddings: torch.Tensor, - attention_mask: torch.Tensor) -> torch.Tensor: - data_type = token_embeddings.dtype - input_mask_expanded = attention_mask.unsqueeze(-1).expand( - token_embeddings.size()).to(data_type) - return torch.sum(token_embeddings * input_mask_expanded, - 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) - - def forward(self, input_ids: torch.Tensor, - att_mask: torch.Tensor) -> torch.Tensor: - with self.autocast_context: - bert_out = self.llm(input_ids=input_ids, attention_mask=att_mask) - - # First element of model_output contains all token embeddings - token_embeddings = bert_out[0] - sentence_embeddings = self.mean_pooling(token_embeddings, att_mask) - sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) - return sentence_embeddings - - -def text2embedding( - model: SentenceTransformer, - text: List[str], - batch_size: Optional[int] = 256, - device: Optional[torch.device] = None, - truncate_long_strs=True, -) -> torch.Tensor: - try: - encoding = model.tokenizer(text, padding=True, - truncation=truncate_long_strs, - return_tensors="pt") - data_len = encoding.input_ids.size(0) - num_full_batches = data_len // batch_size - all_embeddings_list = [] - - # Iterate through batches - if device is None: - device = model.llm_device - with torch.no_grad(): - left_ptr = 0 - for i in range(num_full_batches): - # Forward pass - embeddings = model( - input_ids=encoding.input_ids[left_ptr:left_ptr + - batch_size].to(device), - att_mask=encoding.attention_mask[left_ptr:left_ptr + - batch_size].to(device)) - left_ptr += batch_size - # Append the embeddings to the list - all_embeddings_list.append(embeddings) - # final batch if len not divisible by batch_size - if data_len % batch_size != 0: - embeddings = model( - input_ids=encoding.input_ids[left_ptr:].to(device), - att_mask=encoding.attention_mask[left_ptr:].to(device)) - all_embeddings_list.append(embeddings) - # Concatenate the embeddings from all batches - all_embeddings = torch.cat(all_embeddings_list, dim=0).cpu() - except: # noqa - print("text embedding failed, returning torch.zeros((0, 1024))...") - return torch.zeros((0, 1024)) - - return all_embeddings From 3fb21f346ab9c7895f3fef79749ced1dd6290368 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Thu, 30 May 2024 19:04:34 -0700 Subject: [PATCH 480/752] tiny llama for demo too --- examples/llm_plus_gnn/g_retriever.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3040d4f49c03..526f60040445 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -263,7 +263,7 @@ def adjust_learning_rate(param_group, LR, epoch): def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size, loss_fn, inference_fn, - skip_pretrained_LLM=False): + skip_pretrained_LLM=False, tiny_llama=False): if not skip_pretrained_LLM: print("First comparing against a pretrained LLM...") # Step 1: Define a single batch size test loader @@ -272,7 +272,12 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, # batch size 1 loader for simplicity loader = DataLoader(test_dataset, batch_size=1, drop_last=False, pin_memory=True, shuffle=False) - pure_llm = LLM() + if tiny_llama: + pure_llm = LLM(model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_params=1, + ) + else: + pure_llm = LLM() if path.exists("demo_save_dict.pt"): print("Saved demo outputs for LLM and GNN+LLM found.") print("Would you like to redo untuned LLM eval?") @@ -454,4 +459,4 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, - inference_step) + inference_step, tiny_llama=args.tiny_llama) From fcbe9459639594378cfc5d56be395576e8437aaf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 02:05:36 +0000 Subject: [PATCH 481/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 526f60040445..f1b6b070ecad 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -273,9 +273,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, loader = DataLoader(test_dataset, batch_size=1, drop_last=False, pin_memory=True, shuffle=False) if tiny_llama: - pure_llm = LLM(model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - num_params=1, - ) + pure_llm = LLM( + model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_params=1, + ) else: pure_llm = LLM() if path.exists("demo_save_dict.pt"): From 5f958f5bf147074bece9718e2af21292e35e8ac0 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Fri, 31 May 2024 08:18:35 -0700 Subject: [PATCH 482/752] user interface improvement --- examples/llm_plus_gnn/g_retriever.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 526f60040445..ee468b31fbfb 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -279,8 +279,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, else: pure_llm = LLM() if path.exists("demo_save_dict.pt"): - print("Saved demo outputs for LLM and GNN+LLM found.") - print("Would you like to redo untuned LLM eval?") + print("Saved outputs for the first step of the demo found.") + print("Would you like to redo?") user_input = str(input("(y/n):")).lower() skip_step_one = user_input == "n" else: @@ -294,8 +294,11 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, gnn_llm_preds = [] for out in gnn_llm_eval_outs: gnn_llm_preds += out['pred'] - print( - "Checking pretrained LLM vs trained GNN+LLM for hallucinations...") + if skip_pretrained_LLM: + print("Checking GNN+LLM for hallucinations...") + else: + print("Checking pretrained LLM vs trained\ + GNN+LLM for hallucinations...") for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] From d86bde06ebd4a95d3533301ee79faa9323f04756 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 4 Jun 2024 09:45:19 -0700 Subject: [PATCH 483/752] minor cleanup --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 58625ff8de1b..f22313f35d1c 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -314,8 +314,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_pred = pure_llm_out['pred'][0] pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [(pure_llm_pred, - pure_llm_hallucinates)] + untuned_llm_save_list += [(pure_llm_pred, + pure_llm_hallucinates)] gnn_llm_pred = gnn_llm_preds[i] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, From 4cb83ff99fde4df2f85f7dd700b407c315d3c1b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:46:38 +0000 Subject: [PATCH 484/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f22313f35d1c..09869355e7e6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -314,8 +314,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_pred = pure_llm_out['pred'][0] pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [(pure_llm_pred, - pure_llm_hallucinates)] + untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] gnn_llm_pred = gnn_llm_preds[i] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, From 8f2690bdf7671630c9ac7702765fca5b2c83f98f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 6 Jun 2024 10:35:53 -0700 Subject: [PATCH 485/752] fixing typo --- torch_geometric/datasets/web_qsp_dataset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5a13fcefd904..0b1e512ab59c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -241,11 +241,11 @@ def process(self) -> None: label = ("|").join(data_i["answer"]).lower() raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(nodes)).to("cpu") - psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + pcst_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], nodes, edges, topk=3, topk_e=5, cost_e=0.5) - psct_subgraph["question"] = question - psct_subgraph["label"] = label - psct_subgraph["desc"] = desc - list_of_graphs.append(psct_subgraph.to("cpu")) + pcst_subgraph["question"] = question + pcst_subgraph["label"] = label + pcst_subgraph["desc"] = desc + list_of_graphs.append(pcst_subgraph.to("cpu")) self.save(list_of_graphs, self.processed_paths[0]) From c2b86d75748c11f914ddb916fc3e77ef108dc219 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 6 Jun 2024 15:28:25 -0700 Subject: [PATCH 486/752] user friendliness cleanup --- examples/llm_plus_gnn/g_retriever.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 09869355e7e6..27b7ef4acf1a 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -432,6 +432,9 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "--tiny_llama", action="store_true", help="This example uses LLAMA2 (7B) by default. \ This flag will run the example with TinyLLAMA (1B).") + parser.add_argument( + "--skip_pretrained_llm_eval", action="store_true", + help="This flag will skip the evaluation of the pretrained LLM.") args = parser.parse_args() # check if saved model @@ -462,4 +465,6 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, - inference_step, tiny_llama=args.tiny_llama) + inference_step, + skip_pretrained_LLM=args.skip_pretrained_llm_eval, + tiny_llama=args.tiny_llama) From dfdc5f4409bd8f924907334f578e55289832a5b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 22:29:27 +0000 Subject: [PATCH 487/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 27b7ef4acf1a..3e86cc322489 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -465,6 +465,6 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, - inference_step, + inference_step, skip_pretrained_LLM=args.skip_pretrained_llm_eval, tiny_llama=args.tiny_llama) From 4b2f2a257aaf57796ea29caea8d8ea9dc3a634a5 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 11 Jun 2024 13:10:54 -0700 Subject: [PATCH 488/752] minor bug fix --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ce6d2c5c6759..b28fdb9acfd4 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -300,7 +300,7 @@ def inference( with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, - max_new_tokens=max_new_tokens, + max_new_tokens=max_out_tokens, attention_mask=attention_mask, # do_sample=True, use_cache=True # IMPORTANT! From 9012176e21ac05d3d695620f8f2b4c8ab67a5757 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 22:13:56 +0000 Subject: [PATCH 489/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 12 +++++++----- torch_geometric/nn/models/g_retriever.py | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ee3cfcc695d9..0186d3fa4b83 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -298,7 +298,9 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if skip_pretrained_LLM: print("Checking GNN+LLM for hallucinations...") else: - print("Checking pretrained LLM vs trained GNN+LLM for hallucinations...") # noqa + print( + "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." + ) # noqa for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] @@ -362,10 +364,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, if retrain: print("Finetuning LLM...") since = time.time() - _, _, pure_llm_eval_outputs = train(since, epochs, None, None, batch_size, - eval_batch_size, lr, loss_fn, - inference_fn, model=pure_llm, - dataset=dataset) + _, _, pure_llm_eval_outputs = train(since, epochs, None, None, + batch_size, eval_batch_size, lr, + loss_fn, inference_fn, + model=pure_llm, dataset=dataset) e2e_time = round(time.time() - since, 2) print("E2E time (e2e_time) =", e2e_time, "seconds") else: diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2a463a985c76..e69481f0cc99 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -80,7 +80,8 @@ def __init__( prepare_model_for_kbit_training, ) print("Training our LLM with LORA!") - self.llm_generator = prepare_model_for_kbit_training(self.llm_generator) + self.llm_generator = prepare_model_for_kbit_training( + self.llm_generator) lora_r: int = 8 lora_alpha: int = 16 lora_dropout: float = 0.05 @@ -108,7 +109,7 @@ def __init__( heads=num_gnn_heads, norm='batch_norm', ).to(self.llm_device) - except: # noqa + except: # noqa # to handle gnns that do not have `heads` param self.graph_encoder = gnn_to_use( in_channels=gnn_in_channels, From 0979a5c4edc69523cb5f0397154c2efc26bfa6d9 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:15:36 -0700 Subject: [PATCH 490/752] final super minor cleanup to finish addressing nit --- torch_geometric/datasets/web_qsp_dataset.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2c04edfed992..317d779d7c6e 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -47,10 +47,6 @@ def retrieval_via_pcst( edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc - root = -1 # unrooted - num_clusters = 1 - pruning = "gw" - verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) topk = min(topk, graph.num_nodes) From 43b17219e86401e12e2c932524490ea5810a2a80 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:31:23 -0700 Subject: [PATCH 491/752] fixing my mistake in merging conflicts --- torch_geometric/nn/nlp/sentence_transformer.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 5ea1d3f50b3a..51eae795fcee 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -13,26 +13,13 @@ class PoolingStrategy(Enum): class SentenceTransformer(torch.nn.Module): -<<<<<<< HEAD - r"""Embeds text as a vector using Huggingface transformers. -======= def __init__( self, model_name: str, pooling_strategy: Union[PoolingStrategy, str] = 'mean', ) -> None: super().__init__() ->>>>>>> d6147b451208ab7b228aba0b393bb078b9f04a44 - model_name (str): A string representing the huggingface model to use. - (default: :obj:`"sentence-transformers/all-roberta-large-v1"`) - """ - def __init__( - self, - model_name: Optional[ - str] = "sentence-transformers/all-roberta-large-v1", - ) -> None: - super().__init__() self.model_name = model_name self.pooling_strategy = PoolingStrategy(pooling_strategy) @@ -67,9 +54,8 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: - if len(text) == 0: - return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size + embs: List[Tensor] = [] for start in range(0, len(text), batch_size): token = self.tokenizer( @@ -105,4 +91,4 @@ def last_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: return emb[:, -1] seq_indices = attention_mask.sum(dim=1) - 1 - return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] + return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] \ No newline at end of file From c7f3fad6d8cb411d2b566d300eaaddc105034ea2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 22:32:27 +0000 Subject: [PATCH 492/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/sentence_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 51eae795fcee..cfb26176fcc6 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -91,4 +91,4 @@ def last_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: return emb[:, -1] seq_indices = attention_mask.sum(dim=1) - 1 - return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] \ No newline at end of file + return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] From 21d1896112442ca41e952f03e102476d858fe7db Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:34:33 -0700 Subject: [PATCH 493/752] fixing my mistake in merging conflicts --- torch_geometric/nn/nlp/sentence_transformer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 51eae795fcee..954fad806906 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -15,7 +15,8 @@ class PoolingStrategy(Enum): class SentenceTransformer(torch.nn.Module): def __init__( self, - model_name: str, + model_name: Optional[ + str] = "sentence-transformers/all-roberta-large-v1", pooling_strategy: Union[PoolingStrategy, str] = 'mean', ) -> None: super().__init__() From be267e7c555f21295908932f9e3ed787a7f1a5f8 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:41:36 -0700 Subject: [PATCH 494/752] fixing my mistake in merging conflicts --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 317d779d7c6e..f36ba2b214a7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -47,6 +47,10 @@ def retrieval_via_pcst( edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) topk = min(topk, graph.num_nodes) @@ -102,8 +106,8 @@ def retrieval_via_pcst( costs = np.array(costs + virtual_costs) edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast(edges, prizes, costs, root=-1, num_clusters=1, - pruning="gw", verbosity_level=0) + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] From 89802e9d532e984f233ef1d432ad33eb38130ef0 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:45:38 -0700 Subject: [PATCH 495/752] fixing my mistake in merging conflicts --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f36ba2b214a7..317d779d7c6e 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -47,10 +47,6 @@ def retrieval_via_pcst( edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc - root = -1 # unrooted - num_clusters = 1 - pruning = "gw" - verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) topk = min(topk, graph.num_nodes) @@ -106,8 +102,8 @@ def retrieval_via_pcst( costs = np.array(costs + virtual_costs) edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, - pruning, verbosity_level) + vertices, edges = pcst_fast(edges, prizes, costs, root=-1, num_clusters=1, + pruning="gw", verbosity_level=0) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] From 8811260e9174cfe33f8c18d7532953d0ffba16ab Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 15:49:10 -0700 Subject: [PATCH 496/752] fixing my mistake in merging conflicts --- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 317d779d7c6e..f36ba2b214a7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -47,6 +47,10 @@ def retrieval_via_pcst( edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 if topk > 0: n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) topk = min(topk, graph.num_nodes) @@ -102,8 +106,8 @@ def retrieval_via_pcst( costs = np.array(costs + virtual_costs) edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast(edges, prizes, costs, root=-1, num_clusters=1, - pruning="gw", verbosity_level=0) + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] From 757b63fca6df95181fce070e144a220e4cb23b87 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 16:36:25 -0700 Subject: [PATCH 497/752] recent upstream sentencetransformer has issues, looking how to fix. reverting first --- .../nn/nlp/sentence_transformer.py | 49 +++++-------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 0ec5c7ae999e..59a74d080740 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -1,46 +1,38 @@ -from enum import Enum -from typing import List, Optional, Union +from typing import List, Optional import torch import torch.nn.functional as F from torch import Tensor -class PoolingStrategy(Enum): - MEAN = 'mean' - LAST = 'last' - CLS = 'cls' - - class SentenceTransformer(torch.nn.Module): + r"""Embeds text as a vector using Huggingface transformers. + + model_name (str): A string representing the huggingface model to use. + (default: :obj:`"sentence-transformers/all-roberta-large-v1"`) + """ def __init__( self, model_name: Optional[ str] = "sentence-transformers/all-roberta-large-v1", - pooling_strategy: Union[PoolingStrategy, str] = 'mean', ) -> None: super().__init__() - self.model_name = model_name - self.pooling_strategy = PoolingStrategy(pooling_strategy) from transformers import AutoModel, AutoTokenizer self.tokenizer = AutoTokenizer.from_pretrained(model_name) self.model = AutoModel.from_pretrained(model_name) + def mean_pooling(self, emb: Tensor, attention_mask: Tensor) -> Tensor: + mask = attention_mask.unsqueeze(-1).expand(emb.size()).to(emb.dtype) + return (emb * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) + def forward(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor: out = self.model(input_ids=input_ids, attention_mask=attention_mask) emb = out[0] # First element contains all token embeddings. - if self.pooling_strategy == PoolingStrategy.MEAN: - emb = mean_pooling(emb, attention_mask) - elif self.pooling_strategy == PoolingStrategy.LAST: - emb = last_pooling(emb, attention_mask) - else: - assert self.pooling_strategy == PoolingStrategy.CLS - emb = emb[:, 0, :] - + emb = self.mean_pooling(emb, attention_mask) emb = F.normalize(emb, p=2, dim=1) return emb @@ -55,8 +47,9 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: + if len(text) == 0: + return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size - embs: List[Tensor] = [] for start in range(0, len(text), batch_size): token = self.tokenizer( @@ -77,19 +70,3 @@ def encode( def __repr__(self) -> str: return f'{self.__class__.__name__}(model_name={self.model_name})' - - -def mean_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: - mask = attention_mask.unsqueeze(-1).expand(emb.size()).to(emb.dtype) - return (emb * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) - - -def last_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: - # Check whether language model uses left padding, - # which is always used for decoder LLMs - left_padding = attention_mask[:, -1].sum() == attention_mask.size(0) - if left_padding: - return emb[:, -1] - - seq_indices = attention_mask.sum(dim=1) - 1 - return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] From 1ad05562f50b184fa135e893c372b1ed86d91fe3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 16:43:35 -0700 Subject: [PATCH 498/752] recent upstream sentencetransformer has issues, looking how to fix. reverting first --- .../nn/nlp/sentence_transformer.py | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 59a74d080740..0ec5c7ae999e 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -1,38 +1,46 @@ -from typing import List, Optional +from enum import Enum +from typing import List, Optional, Union import torch import torch.nn.functional as F from torch import Tensor -class SentenceTransformer(torch.nn.Module): - r"""Embeds text as a vector using Huggingface transformers. +class PoolingStrategy(Enum): + MEAN = 'mean' + LAST = 'last' + CLS = 'cls' + - model_name (str): A string representing the huggingface model to use. - (default: :obj:`"sentence-transformers/all-roberta-large-v1"`) - """ +class SentenceTransformer(torch.nn.Module): def __init__( self, model_name: Optional[ str] = "sentence-transformers/all-roberta-large-v1", + pooling_strategy: Union[PoolingStrategy, str] = 'mean', ) -> None: super().__init__() + self.model_name = model_name + self.pooling_strategy = PoolingStrategy(pooling_strategy) from transformers import AutoModel, AutoTokenizer self.tokenizer = AutoTokenizer.from_pretrained(model_name) self.model = AutoModel.from_pretrained(model_name) - def mean_pooling(self, emb: Tensor, attention_mask: Tensor) -> Tensor: - mask = attention_mask.unsqueeze(-1).expand(emb.size()).to(emb.dtype) - return (emb * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) - def forward(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor: out = self.model(input_ids=input_ids, attention_mask=attention_mask) emb = out[0] # First element contains all token embeddings. - emb = self.mean_pooling(emb, attention_mask) + if self.pooling_strategy == PoolingStrategy.MEAN: + emb = mean_pooling(emb, attention_mask) + elif self.pooling_strategy == PoolingStrategy.LAST: + emb = last_pooling(emb, attention_mask) + else: + assert self.pooling_strategy == PoolingStrategy.CLS + emb = emb[:, 0, :] + emb = F.normalize(emb, p=2, dim=1) return emb @@ -47,9 +55,8 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: - if len(text) == 0: - return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size + embs: List[Tensor] = [] for start in range(0, len(text), batch_size): token = self.tokenizer( @@ -70,3 +77,19 @@ def encode( def __repr__(self) -> str: return f'{self.__class__.__name__}(model_name={self.model_name})' + + +def mean_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: + mask = attention_mask.unsqueeze(-1).expand(emb.size()).to(emb.dtype) + return (emb * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) + + +def last_pooling(emb: Tensor, attention_mask: Tensor) -> Tensor: + # Check whether language model uses left padding, + # which is always used for decoder LLMs + left_padding = attention_mask[:, -1].sum() == attention_mask.size(0) + if left_padding: + return emb[:, -1] + + seq_indices = attention_mask.sum(dim=1) - 1 + return emb[torch.arange(emb.size(0), device=emb.device), seq_indices] From f81756e48a260d8877078c8c1e2080880102505b Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 16:46:38 -0700 Subject: [PATCH 499/752] fixed --- torch_geometric/nn/nlp/sentence_transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 0ec5c7ae999e..64353bb63c69 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -55,6 +55,8 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: + if len(text) == 0: + return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size embs: List[Tensor] = [] From 5eb055ab00590bf252f92254f63bf0096f177d6e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Mon, 24 Jun 2024 19:06:22 -0700 Subject: [PATCH 500/752] handling flake8 --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0186d3fa4b83..d3f5f6b09809 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -299,8 +299,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Checking GNN+LLM for hallucinations...") else: print( - "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." - ) # noqa + "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa + ) for i, batch in enumerate(tqdm(loader)): question = batch.question[0] correct_answer = batch.label[0] From 49b25020800e314576b62ed7db22c771a863c7dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 02:07:26 +0000 Subject: [PATCH 501/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index d3f5f6b09809..9045be8ef8cc 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -299,7 +299,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("Checking GNN+LLM for hallucinations...") else: print( - "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa + "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa ) for i, batch in enumerate(tqdm(loader)): question = batch.question[0] From 4743ee4439d042e41970d3a19e440b3e1da09a24 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 25 Jun 2024 11:56:20 -0700 Subject: [PATCH 502/752] added unit testing for llm and gnn+llm --- test/nn/models/test_g_retriever.py | 43 ++++++++++++++++++++++++++++++ test/nn/nlp/test_llm.py | 23 ++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 test/nn/models/test_g_retriever.py diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py new file mode 100644 index 000000000000..9b4e7c4424ef --- /dev/null +++ b/test/nn/models/test_g_retriever.py @@ -0,0 +1,43 @@ +import pytest + +from torch_geometric.testing import onlyFullTest, withCUDA, withPackage +from torch_geometric.nn.models import GRetriever +from torch_geometric.data import Data +import torch + +@withCUDA +@onlyFullTest +@withPackage('transformers') +def test_g_retriever(): + batch = Data() + batch.question = ["Is PyG the best open-source GNN library?"] + batch.label = ["yes!"] + batch.num_nodes = 10 + batch.n_id = torch.arange(10).to(torch.int64) + batch.num_edges = 20 + batch.x = torch.randn((10, 1024)) + batch.edge_attr = torch.randn((20, 1024)) + # Model expects batches sampled from Dataloader + # hardcoding values for single item batch + batch.batch = torch.zeros(batch.num_nodes).to(torch.int64) + batch.ptr = torch.tensor([0, batch.num_nodes]).to(torch.int64) + # simple cycle graph + batch.edge_index = torch.tensor([list(range(10)),list(range(1, 10)) + [0]]) + + + + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + mlp_out_dim=2048, + ) + batch = batch.to(model.llm_device) + # test train + loss = model(batch.question, batch.x, batch.edge_index, batch.batch, + batch.ptr, batch.label, batch.edge_attr) + assert loss is not None + + # test inference + out = model.inference(batch.question, batch.x, batch.edge_index, + batch.batch, batch.ptr, batch.edge_attr) + assert out['pred'] is not None \ No newline at end of file diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index e69de29bb2d1..90d23b8b0561 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -0,0 +1,23 @@ +import pytest + +from torch_geometric.testing import onlyFullTest, withCUDA, withPackage +from torch_geometric.nn.nlp.llm import LLM +from torch_geometric.data import Data + +@withCUDA +@onlyFullTest +@withPackage('transformers') +def test_llm(): + batch = Data() + batch.question = ["Is PyG the best open-source GNN library?"] + batch.label = ["yes!"] + + model = LLM(model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", num_params=1) + + # test train + loss = model(batch.question, batch.label) + assert loss is not None + + # test inference + out = model.inference(batch.question) + assert out['pred'] is not None \ No newline at end of file From 30a2ccdbfd3a27a87718e62d721ec226da15d2f0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 18:58:02 +0000 Subject: [PATCH 503/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/nn/models/test_g_retriever.py | 26 +++++++++++++------------- test/nn/nlp/test_llm.py | 7 ++++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 9b4e7c4424ef..74840cf81f5d 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -1,9 +1,10 @@ import pytest +import torch -from torch_geometric.testing import onlyFullTest, withCUDA, withPackage -from torch_geometric.nn.models import GRetriever from torch_geometric.data import Data -import torch +from torch_geometric.nn.models import GRetriever +from torch_geometric.testing import onlyFullTest, withCUDA, withPackage + @withCUDA @onlyFullTest @@ -22,22 +23,21 @@ def test_g_retriever(): batch.batch = torch.zeros(batch.num_nodes).to(torch.int64) batch.ptr = torch.tensor([0, batch.num_nodes]).to(torch.int64) # simple cycle graph - batch.edge_index = torch.tensor([list(range(10)),list(range(1, 10)) + [0]]) - - + batch.edge_index = torch.tensor( + [list(range(10)), list(range(1, 10)) + [0]]) model = GRetriever( - llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - num_llm_params=1, # 1 Billion - mlp_out_dim=2048, - ) + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + mlp_out_dim=2048, + ) batch = batch.to(model.llm_device) # test train loss = model(batch.question, batch.x, batch.edge_index, batch.batch, - batch.ptr, batch.label, batch.edge_attr) + batch.ptr, batch.label, batch.edge_attr) assert loss is not None # test inference out = model.inference(batch.question, batch.x, batch.edge_index, - batch.batch, batch.ptr, batch.edge_attr) - assert out['pred'] is not None \ No newline at end of file + batch.batch, batch.ptr, batch.edge_attr) + assert out['pred'] is not None diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index 90d23b8b0561..a34842b8d69c 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -1,8 +1,9 @@ import pytest -from torch_geometric.testing import onlyFullTest, withCUDA, withPackage -from torch_geometric.nn.nlp.llm import LLM from torch_geometric.data import Data +from torch_geometric.nn.nlp.llm import LLM +from torch_geometric.testing import onlyFullTest, withCUDA, withPackage + @withCUDA @onlyFullTest @@ -20,4 +21,4 @@ def test_llm(): # test inference out = model.inference(batch.question) - assert out['pred'] is not None \ No newline at end of file + assert out['pred'] is not None From cb27fe1fdefa196f3631c4fac0fd620b1f318f3d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 25 Jun 2024 13:00:34 -0700 Subject: [PATCH 504/752] added unit testing for llm and gnn+llm --- test/nn/models/test_g_retriever.py | 2 +- test/nn/nlp/test_llm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 9b4e7c4424ef..690f0ac271b5 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -8,7 +8,7 @@ @withCUDA @onlyFullTest @withPackage('transformers') -def test_g_retriever(): +def test_g_retriever(device): batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] batch.label = ["yes!"] diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index 90d23b8b0561..3ed79e3dec0e 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -7,7 +7,7 @@ @withCUDA @onlyFullTest @withPackage('transformers') -def test_llm(): +def test_llm(device): batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] batch.label = ["yes!"] From 0ee0362e2fca571516d857794af96e0bfebefded Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 25 Jun 2024 13:12:30 -0700 Subject: [PATCH 505/752] lint clean --- test/nn/models/test_g_retriever.py | 1 - test/nn/nlp/test_llm.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index a22d8c7f062e..45c38ea42f90 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -1,4 +1,3 @@ -import pytest import torch from torch_geometric.data import Data diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index 51f3115c10d9..d0aa016dabf8 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -1,5 +1,3 @@ -import pytest - from torch_geometric.data import Data from torch_geometric.nn.nlp.llm import LLM from torch_geometric.testing import onlyFullTest, withCUDA, withPackage From 8392c889b84a9416531e1d34fb1bede62d7208d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:16:33 +0000 Subject: [PATCH 506/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 223c1668cbfd..53b9c68236da 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -280,4 +280,4 @@ def inference( use_cache=True, ) - return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) \ No newline at end of file + return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) From d99c9acac55508a26996a3ca575683c636179604 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 10:36:21 -0700 Subject: [PATCH 507/752] fixing issues caused by recent merge --- test/nn/models/test_g_retriever.py | 6 +++--- torch_geometric/nn/models/g_retriever.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 45c38ea42f90..bf495a7cce50 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -2,13 +2,13 @@ from torch_geometric.data import Data from torch_geometric.nn.models import GRetriever -from torch_geometric.testing import onlyFullTest, withCUDA, withPackage +from torch_geometric.testing import onlyFullTest, withPackage @withCUDA @onlyFullTest -@withPackage('transformers') -def test_g_retriever(device): +@withPackage('transformers', 'accelerate') +def test_g_retriever() -> None: batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] batch.label = ["yes!"] diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index e69481f0cc99..2fc8e9953eed 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -8,8 +8,8 @@ EOS, IGNORE_INDEX, LLM, - max_new_tokens, - max_txt_len, + MAX_NEW_TOKENS, + MAX_TXT_LEN, ) from torch_geometric.utils import scatter @@ -184,10 +184,10 @@ def forward( for i in range(batch_size): # Add bos & eos token label_input_ids = labels.input_ids[ - i][:max_new_tokens] + eos_tokens.input_ids + i][:MAX_NEW_TOKENS] + eos_tokens.input_ids if additional_text_context is not None: input_ids = context.input_ids[ - i][:max_txt_len] + questions.input_ids[ + i][:MAX_TXT_LEN] + questions.input_ids[ i] + eos_user_tokens.input_ids + label_input_ids else: input_ids = questions.input_ids[ @@ -244,7 +244,7 @@ def inference( ptr: torch.Tensor, edge_attr: Optional[torch.Tensor] = None, additional_text_context: Optional[List[str]] = None, - max_out_tokens: Optional[int] = max_new_tokens, + max_out_tokens: Optional[int] = MAX_NEW_TOKENS, ): r"""Inference. @@ -280,7 +280,7 @@ def inference( # Add bos & eos token if additional_text_context is not None: input_ids = context.input_ids[ - i][:max_txt_len] + questions.input_ids[ + i][:MAX_TXT_LEN] + questions.input_ids[ i] + eos_user_tokens.input_ids else: input_ids = questions.input_ids[i] + eos_user_tokens.input_ids From 1da5476a3c07ea40dfabad27738fb6b1672fb7bb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 10:53:44 -0700 Subject: [PATCH 508/752] fixing issues caused by recent merge --- torch_geometric/nn/models/g_retriever.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2fc8e9953eed..e2c64eb0c954 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -63,14 +63,11 @@ def __init__( ) -> None: super().__init__() if 'llama2-7b' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2-7b', llm_dtype, - num_params=num_llm_params) + self.llm_to_use = LLM('llama2-7b', num_llm_params, llm_dtype) elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma', llm_dtype, - num_params=num_llm_params) + self.llm_to_use = LLM('gemma', num_llm_params, llm_dtype) else: - self.llm_to_use = LLM(llm_to_use, llm_dtype, - num_params=num_llm_params) + self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: From aee71fdec8cee55250a7e27b8eb3cf5cc352538e Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 11:10:03 -0700 Subject: [PATCH 509/752] fixing issues caused by recent merge --- torch_geometric/nn/models/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index e2c64eb0c954..b9edce3ae4f3 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -164,7 +164,7 @@ def forward( to give to the LLM, such as textified knowledge graphs. """ batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(question, additional_text_context) # noqa + bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa # encode labels labels = self.tokenizer(label, add_special_tokens=False) # encode training specific special token @@ -264,7 +264,7 @@ def inference( generate. (default: {32}) """ batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use.encode_inputs(question, additional_text_context) # noqa + bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa # encode graphs graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, batch) From fc66cfd2803d33e40c1d4c3b1a6f4cb72b3bdf0d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 11:17:47 -0700 Subject: [PATCH 510/752] fixing issues caused by recent merge --- examples/llm_plus_gnn/g_retriever.py | 9 ++++----- torch_geometric/nn/models/g_retriever.py | 8 +------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9045be8ef8cc..3a8e602892b5 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -244,9 +244,9 @@ def adjust_learning_rate(param_group, LR, epoch): progress_bar_test = tqdm(range(len(test_loader))) for step, batch in enumerate(test_loader): with torch.no_grad(): - output = inference_fn(model, batch, model_save_name) - output["label"] = batch.label - eval_output.append(output) + pred = inference_fn(model, batch, model_save_name) + eval_data = {"pred": pred, "question":batch.question, "desc": batch.desc, "label":batch.label} + eval_output.append(eval_data) progress_bar_test.update(1) # Step 6 Post-processing & compute metrics @@ -310,9 +310,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, else: # GNN+LLM only using 32 tokens to answer. # Allow more output tokens for untrained LLM - pure_llm_out = pure_llm.inference(batch.question, batch.desc, + pure_llm_pred = pure_llm.inference(batch.question, batch.desc, max_out_tokens=256) - pure_llm_pred = pure_llm_out['pred'][0] pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index b9edce3ae4f3..d68e9b8f61d8 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -313,13 +313,7 @@ def inference( # do_sample=True, use_cache=True # IMPORTANT! ) - pred = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - - return { - 'pred': pred, - 'question': question, - 'desc': additional_text_context, - } + return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) def print_trainable_params(self) -> None: trainable_params = 0 From 46dd7637272cfd45c287f6a36ef7ce732e382251 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 18:19:03 +0000 Subject: [PATCH 511/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3a8e602892b5..0724b1959dc3 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -245,7 +245,12 @@ def adjust_learning_rate(param_group, LR, epoch): for step, batch in enumerate(test_loader): with torch.no_grad(): pred = inference_fn(model, batch, model_save_name) - eval_data = {"pred": pred, "question":batch.question, "desc": batch.desc, "label":batch.label} + eval_data = { + "pred": pred, + "question": batch.question, + "desc": batch.desc, + "label": batch.label + } eval_output.append(eval_data) progress_bar_test.update(1) @@ -311,7 +316,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, # GNN+LLM only using 32 tokens to answer. # Allow more output tokens for untrained LLM pure_llm_pred = pure_llm.inference(batch.question, batch.desc, - max_out_tokens=256) + max_out_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] From 186ce37aef595fa9cd79e22a1f5daac151c9b76d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 11:33:24 -0700 Subject: [PATCH 512/752] fixing issues caused by recent merge --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3a8e602892b5..73e394dac882 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -278,7 +278,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, num_params=1, ) else: - pure_llm = LLM() + pure_llm = LLM(model_name="llama2-7b", num_params=7) if path.exists("demo_save_dict.pt"): print("Saved outputs for the first step of the demo found.") print("Would you like to redo?") From a0e25c158e4a459e01973fd7acb33dc922d3c1ad Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 11:55:03 -0700 Subject: [PATCH 513/752] fixed issues. checking if bos_token makes any diff for g-retriever mirroring matthias's change on llm --- torch_geometric/nn/models/g_retriever.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index d68e9b8f61d8..2fffd971a4f1 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -304,10 +304,14 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - + bos_token = self.tokenizer( + BOS, + add_special_tokens=False, + ).input_ids[0] with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, + bos_token_id=bos_token, max_new_tokens=max_out_tokens, attention_mask=attention_mask, # do_sample=True, From ea357c6c098faec9ea58c892d7f8754f3b3c5ec3 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 12:06:43 -0700 Subject: [PATCH 514/752] fixed issues. checking if bos_token makes any diff for g-retriever mirroring matthias's change on llm --- torch_geometric/nn/models/g_retriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 2fffd971a4f1..ac8a5940e592 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -6,6 +6,7 @@ from torch_geometric.nn.models import GAT from torch_geometric.nn.nlp.llm import ( EOS, + BOS, IGNORE_INDEX, LLM, MAX_NEW_TOKENS, From 1647eddb1078995970291c748efedfb5415b2d1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:08:00 +0000 Subject: [PATCH 515/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ac8a5940e592..413fec311195 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -5,8 +5,8 @@ from torch_geometric.nn.models import GAT from torch_geometric.nn.nlp.llm import ( - EOS, BOS, + EOS, IGNORE_INDEX, LLM, MAX_NEW_TOKENS, From 00a24104e185fbf050825e6089039e00aa6c7838 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 12:19:04 -0700 Subject: [PATCH 516/752] reverting, made no diff --- torch_geometric/nn/models/g_retriever.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ac8a5940e592..a32a20d6a0e1 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -6,7 +6,6 @@ from torch_geometric.nn.models import GAT from torch_geometric.nn.nlp.llm import ( EOS, - BOS, IGNORE_INDEX, LLM, MAX_NEW_TOKENS, @@ -305,14 +304,10 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - bos_token = self.tokenizer( - BOS, - add_special_tokens=False, - ).input_ids[0] + with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, - bos_token_id=bos_token, max_new_tokens=max_out_tokens, attention_mask=attention_mask, # do_sample=True, From d05b6b269d9301202137db275eb2c2bdfe3b85a8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:20:59 +0000 Subject: [PATCH 517/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index a32a20d6a0e1..d68e9b8f61d8 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -304,7 +304,7 @@ def inference( inputs_embeds = torch.stack(batch_inputs_embeds, dim=0).to(self.llm_device) attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - + with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, From 96c287763d53adf7c5f55e112af5bcd19cfed0fb Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 13:31:53 -0700 Subject: [PATCH 518/752] reverting, made no diff --- test/nn/models/test_g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index bf495a7cce50..e39ffafad092 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -5,7 +5,6 @@ from torch_geometric.testing import onlyFullTest, withPackage -@withCUDA @onlyFullTest @withPackage('transformers', 'accelerate') def test_g_retriever() -> None: From c1e1b2f91577bcf08266d13c2dfaadcaf4caab4d Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 13:49:31 -0700 Subject: [PATCH 519/752] breaking PR https://github.com/pyg-team/pytorch_geometric/pull/9167/ down further, focusing on G-retriever model this time --- test/nn/models/test_g_retriever.py | 41 +++ torch_geometric/nn/models/__init__.py | 2 + torch_geometric/nn/models/g_retriever.py | 328 +++++++++++++++++++++++ 3 files changed, 371 insertions(+) create mode 100644 test/nn/models/test_g_retriever.py create mode 100644 torch_geometric/nn/models/g_retriever.py diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py new file mode 100644 index 000000000000..e39ffafad092 --- /dev/null +++ b/test/nn/models/test_g_retriever.py @@ -0,0 +1,41 @@ +import torch + +from torch_geometric.data import Data +from torch_geometric.nn.models import GRetriever +from torch_geometric.testing import onlyFullTest, withPackage + + +@onlyFullTest +@withPackage('transformers', 'accelerate') +def test_g_retriever() -> None: + batch = Data() + batch.question = ["Is PyG the best open-source GNN library?"] + batch.label = ["yes!"] + batch.num_nodes = 10 + batch.n_id = torch.arange(10).to(torch.int64) + batch.num_edges = 20 + batch.x = torch.randn((10, 1024)) + batch.edge_attr = torch.randn((20, 1024)) + # Model expects batches sampled from Dataloader + # hardcoding values for single item batch + batch.batch = torch.zeros(batch.num_nodes).to(torch.int64) + batch.ptr = torch.tensor([0, batch.num_nodes]).to(torch.int64) + # simple cycle graph + batch.edge_index = torch.tensor( + [list(range(10)), list(range(1, 10)) + [0]]) + + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + mlp_out_dim=2048, + ) + batch = batch.to(model.llm_device) + # test train + loss = model(batch.question, batch.x, batch.edge_index, batch.batch, + batch.ptr, batch.label, batch.edge_attr) + assert loss is not None + + # test inference + out = model.inference(batch.question, batch.x, batch.edge_index, + batch.batch, batch.ptr, batch.edge_attr) + assert out['pred'] is not None diff --git a/torch_geometric/nn/models/__init__.py b/torch_geometric/nn/models/__init__.py index 334970da5c62..7cfadf0143b2 100644 --- a/torch_geometric/nn/models/__init__.py +++ b/torch_geometric/nn/models/__init__.py @@ -28,6 +28,7 @@ from .pmlp import PMLP from .neural_fingerprint import NeuralFingerprint from .visnet import ViSNet +from .g_retriever import GRetriever # Deprecated: from torch_geometric.explain.algorithm.captum import (to_captum_input, @@ -75,4 +76,5 @@ 'PMLP', 'NeuralFingerprint', 'ViSNet', + 'GRetriever', ] diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py new file mode 100644 index 000000000000..d68e9b8f61d8 --- /dev/null +++ b/torch_geometric/nn/models/g_retriever.py @@ -0,0 +1,328 @@ +from typing import List, Optional + +import torch +import torch.nn as nn + +from torch_geometric.nn.models import GAT +from torch_geometric.nn.nlp.llm import ( + EOS, + IGNORE_INDEX, + LLM, + MAX_NEW_TOKENS, + MAX_TXT_LEN, +) +from torch_geometric.utils import scatter + + +class GRetriever(nn.Module): + r"""This GNN+LLM implementation is based on G-retriever. + Original Paper: `_. + See `examples/llm_plus_gnn/g_retriever.py` for example usage. + + Args: + llm_to_use (str): A string representing the huggingface model you + want to use. This module has been tested for 'llama2' and 'gemma'. + Other huggingface transformer models should work if you pass the + correct name, see huggingface.co for details. If any issues occur + please file an issue on + https://github.com/pyg-team/pytorch_geometric + and assign to puririshi98. (default: :obj:'llama2') + llm_use_lora (bool): use LORA from peft for training the LLM. see + https://huggingface.co/docs/peft/en/index for details. + llm_dtype (torch.dtype): The dtype to use for the LLM. + (default :obj: `torch.bloat16`) + num_llm_params (int): An integer representing how many params your + huggingface transformer model has, in billions. This is used to + automatically allocate the number of gpus needed, given the + available GPU memory of your GPUs (default :obj:`7`) + gnn_to_use (BasicGNN): Please pass a valid model that extends + torch_geometric.nn.models.basic_gnn.BasicGNN. (default: :obj:`GAT`) + gnn_in_channels (int): (default: 1024) + gnn_hidden_channels (int): (default: 1024) + gnn_out_channels (int): (default: 1024) + num_gnn_layers (int): (default: 4) + num_gnn_heads (int): Number of heads to use for BasicGNNs with the + `heads` kwarg. (default: 4) + mlp_hidden_dim (int): (default: 2048) + mlp_out_dim (int): (default: 4096) + """ + def __init__( + self, + llm_to_use='llama2-7b', + llm_use_lora: bool = False, + llm_dtype=torch.bfloat16, + num_llm_params: int = 7, + gnn_to_use=GAT, + gnn_in_channels: int = 1024, + gnn_hidden_channels: int = 1024, + gnn_out_channels: int = 1024, + num_gnn_layers: int = 4, + num_gnn_heads: int = 4, + mlp_hidden_dim: int = 2048, + mlp_out_dim: int = 4096, + ) -> None: + super().__init__() + if 'llama2-7b' in llm_to_use.lower(): + self.llm_to_use = LLM('llama2-7b', num_llm_params, llm_dtype) + elif 'gemma' in llm_to_use.lower(): + self.llm_to_use = LLM('gemma', num_llm_params, llm_dtype) + else: + self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) + self.llm_generator = self.llm_to_use.llm + self.llm_dtype = llm_dtype + if llm_use_lora: + from peft import ( + LoraConfig, + get_peft_model, + prepare_model_for_kbit_training, + ) + print("Training our LLM with LORA!") + self.llm_generator = prepare_model_for_kbit_training( + self.llm_generator) + lora_r: int = 8 + lora_alpha: int = 16 + lora_dropout: float = 0.05 + lora_target_modules = [ + "q_proj", + "v_proj", + ] + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + self.llm_generator = get_peft_model(self.llm_generator, config) + self.llm_device = self.llm_to_use.llm_device + self.tokenizer = self.llm_to_use.tokenizer + try: + self.graph_encoder = gnn_to_use( + in_channels=gnn_in_channels, + out_channels=gnn_out_channels, + hidden_channels=gnn_hidden_channels, + num_layers=num_gnn_layers, + heads=num_gnn_heads, + norm='batch_norm', + ).to(self.llm_device) + except: # noqa + # to handle gnns that do not have `heads` param + self.graph_encoder = gnn_to_use( + in_channels=gnn_in_channels, + out_channels=gnn_out_channels, + hidden_channels=gnn_hidden_channels, + num_layers=num_gnn_layers, + norm='batch_norm', + ).to(self.llm_device) + # For the MLP Projection + mlp_hidden_dim = gnn_out_channels + self.projector = nn.Sequential( + nn.Linear(gnn_out_channels, mlp_hidden_dim), + nn.Sigmoid(), + nn.Linear(mlp_hidden_dim, mlp_out_dim), + ).to(self.llm_device) + + self.word_embedding = self.llm_to_use.word_embedding + + def encode_graphs(self, node_feat, edge_index, edge_attr, batch): + x = node_feat.to(self.llm_device) + edge_index = edge_index.long().to(self.llm_device) + edge_attr = edge_attr.to(self.llm_device) + n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) + batch = batch.to(self.llm_device) + # mean pooling + g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') + return g_embeds + + def forward( + self, + question: List[str], + node_feat: torch.Tensor, + edge_index: torch.Tensor, + batch: torch.Tensor, + ptr: torch.Tensor, + label: List[str], + edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None, + ): + r"""Forward pass. + + Args: + question (List[str]): The questions/prompts. + node_feat (torch.Tensor): The input node features. + edge_index (torch.Tensor or SparseTensor): The edge indices. + batch (torch.Tensor): The batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns + each element to a specific example. + ptr (torch.Tensor): The pointer vector, denoting the + boundaries between examples in the batch. + label (List[str]): The answers/labels. + edge_attr (torch.Tensor, optional): The edge features (if supported + by the GNN being used). (default: :obj:`None`) + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa + # encode labels + labels = self.tokenizer(label, add_special_tokens=False) + # encode training specific special token + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + + # encode graphs + graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, + batch) + graph_embeds = self.projector(graph_embeds) + batch_inputs_embeds = [] + batch_attention_mask = [] + batch_label_input_ids = [] + num_nodes_per_graph = ptr[1:] - ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + label_input_ids = labels.input_ids[ + i][:MAX_NEW_TOKENS] + eos_tokens.input_ids + if additional_text_context is not None: + input_ids = context.input_ids[ + i][:MAX_TXT_LEN] + questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + else: + input_ids = questions.input_ids[ + i] + eos_user_tokens.input_ids + label_input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + label_input_ids = [IGNORE_INDEX + ] * (inputs_embeds.shape[0] - + len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat([ + pad_embeds.repeat(pad_length, 1).to(self.llm_device), + batch_inputs_embeds[i].to(self.llm_device) + ]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + batch_label_input_ids[ + i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + label_input_ids = torch.tensor(batch_label_input_ids).to( + self.llm_device) + with self.llm_to_use.autocast_context: + outputs = self.llm_generator( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + return_dict=True, + labels=label_input_ids, + ) + return outputs.loss + + @torch.no_grad() + def inference( + self, + question: List[str], + node_feat: torch.Tensor, + edge_index: torch.Tensor, + batch: torch.Tensor, + ptr: torch.Tensor, + edge_attr: Optional[torch.Tensor] = None, + additional_text_context: Optional[List[str]] = None, + max_out_tokens: Optional[int] = MAX_NEW_TOKENS, + ): + r"""Inference. + + Args: + question (List[str]): The questions/prompts. + node_feat (torch.Tensor): The input node features. + edge_index (torch.Tensor or SparseTensor): The edge indices. + edge_weight (torch.Tensor, optional): The edge weights (if + supported by the underlying GNN layer). (default: :obj:`None`) + batch (torch.Tensor): The batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns + each element to a specific example. + ptr (torch.Tensor): The pointer vector, denoting the + boundaries between examples in the batch. + edge_attr (torch.Tensor, optional): The edge features (if supported + by the GNN being used). (default: :obj:`None`) + additional_text_context (List[str], optional): Additional context + to give to the LLM, such as textified knowledge graphs. + max_out_tokens (int, optional): How many tokens for the LLM to + generate. (default: {32}) + """ + batch_size, questions, context, eos_user_tokens, \ + bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa + # encode graphs + graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, + batch) + graph_embeds = self.projector(graph_embeds) + + batch_inputs_embeds = [] + batch_attention_mask = [] + num_nodes_per_graph = ptr[1:] - ptr[:-1] + for i in range(batch_size): + # Add bos & eos token + if additional_text_context is not None: + input_ids = context.input_ids[ + i][:MAX_TXT_LEN] + questions.input_ids[ + i] + eos_user_tokens.input_ids + else: + input_ids = questions.input_ids[i] + eos_user_tokens.input_ids + inputs_embeds = self.word_embedding( + torch.tensor(input_ids).to(self.llm_device)) + to_cat = [bos_embeds] + if num_nodes_per_graph[i] != 0: + to_cat.append(graph_embeds[i].unsqueeze(0)) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + dim=0) + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.shape[0]) + + # pad inputs_embeds + max_length = max([x.shape[0] for x in batch_inputs_embeds]) + for i in range(batch_size): + pad_length = max_length - batch_inputs_embeds[i].shape[0] + batch_inputs_embeds[i] = torch.cat( + [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) + batch_attention_mask[i] = [0 + ] * pad_length + batch_attention_mask[i] + + inputs_embeds = torch.stack(batch_inputs_embeds, + dim=0).to(self.llm_device) + attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) + + with self.llm_to_use.autocast_context: + outputs = self.llm_generator.generate( + inputs_embeds=inputs_embeds, + max_new_tokens=max_out_tokens, + attention_mask=attention_mask, + # do_sample=True, + use_cache=True # IMPORTANT! + ) + return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + + def print_trainable_params(self) -> None: + trainable_params = 0 + all_param = 0 + for _, param in self.named_parameters(): + num_params = param.numel() + + all_param += num_params + if param.requires_grad: + trainable_params += num_params + + return trainable_params, all_param From d8a72ed291855148c1f7c9dde48c386c3e4e0e90 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 13:53:51 -0700 Subject: [PATCH 520/752] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88aa8222cb81..b392715ecd1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## \[2.6.0\] - 2024-MM-DD ### Added - +- Added the `nn.models.GRetriever` model ([#9480](https://github.com/pyg-team/pytorch_geometric/pull/9480)) - Added the `nn.nlp.LLM` model ([#9462](https://github.com/pyg-team/pytorch_geometric/pull/9462)) - Added an example of training GNNs for a graph-level regression task ([#9070](https://github.com/pyg-team/pytorch_geometric/pull/9070)) - Added `utils.from_rdmol`/`utils.to_rdmol` functionality ([#9452](https://github.com/pyg-team/pytorch_geometric/pull/9452)) From 10ed6e682f70d1d64d82751745169011dad6059c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 20:55:01 +0000 Subject: [PATCH 521/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b392715ecd1f..596bf2b7a769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## \[2.6.0\] - 2024-MM-DD ### Added + - Added the `nn.models.GRetriever` model ([#9480](https://github.com/pyg-team/pytorch_geometric/pull/9480)) - Added the `nn.nlp.LLM` model ([#9462](https://github.com/pyg-team/pytorch_geometric/pull/9462)) - Added an example of training GNNs for a graph-level regression task ([#9070](https://github.com/pyg-team/pytorch_geometric/pull/9070)) From c204bc999893ab40a651f799836cf3eff4f33141 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 14:03:10 -0700 Subject: [PATCH 522/752] integration. Breaking down PR https://github.com/pyg-team/pytorch_geometric/pull/9167/ further --- test/datasets/test_web_qsp_dataset.py | 10 + torch_geometric/datasets/__init__.py | 2 + torch_geometric/datasets/web_qsp_dataset.py | 252 ++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 test/datasets/test_web_qsp_dataset.py create mode 100644 torch_geometric/datasets/web_qsp_dataset.py diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py new file mode 100644 index 000000000000..aec3bcf99438 --- /dev/null +++ b/test/datasets/test_web_qsp_dataset.py @@ -0,0 +1,10 @@ +from torch_geometric.testing import onlyFullTest, onlyOnline + + +@onlyOnline +@onlyFullTest +def test_web_qsp_dataset(get_dataset): + dataset = get_dataset(name='WebQSPDataset') + assert len(dataset) == 4700 + assert str(dataset) == "WebQSPDataset(4700)" + \ No newline at end of file diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index b7700b25c56e..cc8ef498debb 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -76,6 +76,7 @@ from .wikidata import Wikidata5M from .myket import MyketDataset from .brca_tgca import BrcaTcga +from .web_qsp_dataset import WebQSPDataset from .dbp15k import DBP15K from .aminer import AMiner @@ -110,6 +111,7 @@ import torch_geometric.datasets.utils homo_datasets = [ + 'WebQSPDataset', 'KarateClub', 'TUDataset', 'GNNBenchmarkDataset', diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py new file mode 100644 index 000000000000..884673e8c288 --- /dev/null +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -0,0 +1,252 @@ +# Alot of code in this file is based on the original G-Retriever paper +# url: https://arxiv.org/abs/2402.07630 +from typing import Dict, List, Tuple, no_type_check + +import numpy as np +import torch +from tqdm import tqdm + +from torch_geometric.data import Data, InMemoryDataset +from torch_geometric.nn.nlp import SentenceTransformer + +try: + from pandas import DataFrame + WITH_PANDAS = True +except ImportError: + DataFrame = None + WITH_PANDAS = False + +try: + from pcst_fast import pcst_fast + WITH_PCST = True +except ImportError: + WITH_PCST = False + +try: + import datasets + WITH_DATASETS = True +except ImportError: + WITH_DATASETS = False + + +@no_type_check +def retrieval_via_pcst( + graph: Data, + q_emb: torch.Tensor, + textual_nodes: DataFrame, + textual_edges: DataFrame, + topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5, +) -> Tuple[Data, str]: + c = 0.01 + if len(textual_nodes) == 0 or len(textual_edges) == 0: + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + graph = Data(x=graph.x, edge_index=graph.edge_index, + edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + return graph, desc + + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 + if topk > 0: + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) + topk = min(topk, graph.num_nodes) + _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) + + n_prizes = torch.zeros_like(n_prizes) + n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() + else: + n_prizes = torch.zeros(graph.num_nodes) + + if topk_e > 0: + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + topk_e = min(topk_e, e_prizes.unique().size(0)) + + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + e_prizes[e_prizes < topk_e_values[-1]] = 0.0 + last_topk_e_value = topk_e + for k in range(topk_e): + indices = e_prizes == topk_e_values[k] + value = min((topk_e - k) / sum(indices), last_topk_e_value - c) + e_prizes[indices] = value + last_topk_e_value = value * (1 - c) + # reduce the cost of the edges such that at least one edge is selected + cost_e = min(cost_e, e_prizes.max().item() * (1 - c / 2)) + else: + e_prizes = torch.zeros(graph.num_edges) + + costs = [] + edges = [] + virtual_n_prizes = [] + virtual_edges = [] + virtual_costs = [] + mapping_n = {} + mapping_e = {} + for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + prize_e = e_prizes[i] + if prize_e <= cost_e: + mapping_e[len(edges)] = i + edges.append((src, dst)) + costs.append(cost_e - prize_e) + else: + virtual_node_id = graph.num_nodes + len(virtual_n_prizes) + mapping_n[virtual_node_id] = i + virtual_edges.append((src, virtual_node_id)) + virtual_edges.append((virtual_node_id, dst)) + virtual_costs.append(0) + virtual_costs.append(0) + virtual_n_prizes.append(prize_e - cost_e) + + prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) + num_edges = len(edges) + if len(virtual_costs) > 0: + costs = np.array(costs + virtual_costs) + edges = np.array(edges + virtual_edges) + + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) + + selected_nodes = vertices[vertices < graph.num_nodes] + selected_edges = [mapping_e[e] for e in edges if e < num_edges] + virtual_vertices = vertices[vertices >= graph.num_nodes] + if len(virtual_vertices) > 0: + virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges + virtual_edges) + + edge_index = graph.edge_index[:, selected_edges] + selected_nodes = np.unique( + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + + n = textual_nodes.iloc[selected_nodes] + e = textual_edges.iloc[selected_edges] + desc = n.to_csv(index=False) + "\n" + e.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + + mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + + x = graph.x[selected_nodes] + edge_attr = graph.edge_attr[selected_edges] + src = [mapping[i] for i in edge_index[0].tolist()] + dst = [mapping[i] for i in edge_index[1].tolist()] + edge_index = torch.LongTensor([src, dst]) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) + + return data, desc + + +class WebQSPDataset(InMemoryDataset): + r"""The WebQuestionsSP dataset was released as part of + “The Value of Semantic Parse Labeling for Knowledge + Base Question Answering” + [Yih, Richardson, Meek, Chang & Suh, 2016]. + It contains semantic parses, vs. answers, for a set of questions + that originally comes from WebQuestions [Berant et al., 2013]." + Processing based on "G-Retriever: Retrieval-Augmented Generation + for Textual Graph Understanding and Question Answering". + Requires datasets and transformers from HuggingFace. + + Args: + root (str): Root directory where the dataset should be saved. + force_reload (bool, optional): Whether to re-process the dataset. + (default: :obj:`False`) + """ + def __init__( + self, + root: str = "", + force_reload: bool = False, + ) -> None: + missing_str_list = [] + if not WITH_PCST: + missing_str_list.append('pcst_fast') + if not WITH_DATASETS: + missing_str_list.append('datasets') + if not WITH_PANDAS: + missing_str_list.append('pandas') + if len(missing_str_list) > 0: + missing_str = ' '.join(missing_str_list) + error_out = f"`pip install {missing_str}` to use this dataset." + raise ImportError(error_out) + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + super().__init__(root, None, None, force_reload=force_reload) + self.load(self.processed_paths[0]) + + @property + def raw_file_names(self) -> List[str]: + return [] + + @property + def processed_file_names(self) -> List[str]: + return ["list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt"] + + def download(self) -> None: + dataset = datasets.load_dataset("rmanluo/RoG-webqsp") + self.raw_dataset = datasets.concatenate_datasets( + [dataset["train"], dataset["validation"], dataset["test"]]) + self.split_idxs = { + "train": + torch.arange(len(dataset["train"])), + "val": + torch.arange(len(dataset["validation"])) + len(dataset["train"]), + "test": + torch.arange(len(dataset["test"])) + len(dataset["train"]) + + len(dataset["validation"]) + } + + def process(self) -> None: + self.model = SentenceTransformer().to(self.device) + self.model.eval() + self.questions = [i["question"] for i in self.raw_dataset] + list_of_graphs = [] + # encode questions + print("Encoding questions...") + q_embs = self.model.encode(self.questions, batch_size=256) + print("Encoding graphs...") + for index in tqdm(range(len(self.raw_dataset))): + data_i = self.raw_dataset[index] + raw_nodes: Dict[str, int] = {} + raw_edges = [] + for tri in data_i["graph"]: + h, r, t = tri + h = h.lower() + t = t.lower() + if h not in raw_nodes: + raw_nodes[h] = len(raw_nodes) + if t not in raw_nodes: + raw_nodes[t] = len(raw_nodes) + raw_edges.append({ + "src": raw_nodes[h], + "edge_attr": r, + "dst": raw_nodes[t] + }) + nodes = DataFrame([{ + "node_id": v, + "node_attr": k + } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"]) + edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) + # encode nodes + nodes.node_attr = nodes.node_attr.fillna("") + x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) + # encode edges + edge_attr = self.model.encode(edges.edge_attr.tolist(), + batch_size=256) + edge_index = torch.LongTensor( + [edges.src.tolist(), edges.dst.tolist()]) + question = f"Question: {data_i['question']}\nAnswer: " + label = ("|").join(data_i["answer"]).lower() + raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(nodes)).to("cpu") + pcst_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + nodes, edges, topk=3, + topk_e=5, cost_e=0.5) + pcst_subgraph["question"] = question + pcst_subgraph["label"] = label + pcst_subgraph["desc"] = desc + list_of_graphs.append(pcst_subgraph.to("cpu")) + self.save(list_of_graphs, self.processed_paths[0]) \ No newline at end of file From d958410ffd6772288a73f6fb83663e25050d33ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:05:25 +0000 Subject: [PATCH 523/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/datasets/test_web_qsp_dataset.py | 7 +++---- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index aec3bcf99438..9b484da78bda 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -4,7 +4,6 @@ @onlyOnline @onlyFullTest def test_web_qsp_dataset(get_dataset): - dataset = get_dataset(name='WebQSPDataset') - assert len(dataset) == 4700 - assert str(dataset) == "WebQSPDataset(4700)" - \ No newline at end of file + dataset = get_dataset(name='WebQSPDataset') + assert len(dataset) == 4700 + assert str(dataset) == "WebQSPDataset(4700)" diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 884673e8c288..f36ba2b214a7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -249,4 +249,4 @@ def process(self) -> None: pcst_subgraph["label"] = label pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) - self.save(list_of_graphs, self.processed_paths[0]) \ No newline at end of file + self.save(list_of_graphs, self.processed_paths[0]) From 6e9a750950282e23dd51bf4eb4b16df524fc381f Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 14:07:29 -0700 Subject: [PATCH 524/752] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88aa8222cb81..77142f40fb03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## \[2.6.0\] - 2024-MM-DD ### Added - +- Added the `WebQSPDataset` dataset ([#9481](https://github.com/pyg-team/pytorch_geometric/pull/9481)) - Added the `nn.nlp.LLM` model ([#9462](https://github.com/pyg-team/pytorch_geometric/pull/9462)) - Added an example of training GNNs for a graph-level regression task ([#9070](https://github.com/pyg-team/pytorch_geometric/pull/9070)) - Added `utils.from_rdmol`/`utils.to_rdmol` functionality ([#9452](https://github.com/pyg-team/pytorch_geometric/pull/9452)) From ae6befe9d589e1612f29a4802f877560aff5bfed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:09:07 +0000 Subject: [PATCH 525/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77142f40fb03..e5e12ea50048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## \[2.6.0\] - 2024-MM-DD ### Added + - Added the `WebQSPDataset` dataset ([#9481](https://github.com/pyg-team/pytorch_geometric/pull/9481)) - Added the `nn.nlp.LLM` model ([#9462](https://github.com/pyg-team/pytorch_geometric/pull/9462)) - Added an example of training GNNs for a graph-level regression task ([#9070](https://github.com/pyg-team/pytorch_geometric/pull/9070)) From a87777772f0656c6ea3a55c70d50951964868fe2 Mon Sep 17 00:00:00 2001 From: puririshi98 Date: Tue, 2 Jul 2024 14:22:29 -0700 Subject: [PATCH 526/752] minor fix --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f36ba2b214a7..b7db42c3fe27 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -200,7 +200,7 @@ def download(self) -> None: } def process(self) -> None: - self.model = SentenceTransformer().to(self.device) + self.model = SentenceTransformer("sentence-transformers/all-roberta-large-v1").to(self.device) self.model.eval() self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] From c38eedfdec87b8df2d96973f025c49c1a46ddf2f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 14:23:31 -0700 Subject: [PATCH 527/752] cleanup --- torch_geometric/nn/nlp/sentence_transformer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index 64353bb63c69..e102eb037e5b 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -15,8 +15,7 @@ class PoolingStrategy(Enum): class SentenceTransformer(torch.nn.Module): def __init__( self, - model_name: Optional[ - str] = "sentence-transformers/all-roberta-large-v1", + model_name: str, pooling_strategy: Union[PoolingStrategy, str] = 'mean', ) -> None: super().__init__() From 6de4caf1253ec8223f9173e7d439a10b09492cc4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:23:57 +0000 Subject: [PATCH 528/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index b7db42c3fe27..39f74e397f20 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -200,7 +200,8 @@ def download(self) -> None: } def process(self) -> None: - self.model = SentenceTransformer("sentence-transformers/all-roberta-large-v1").to(self.device) + self.model = SentenceTransformer( + "sentence-transformers/all-roberta-large-v1").to(self.device) self.model.eval() self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] From 8e35f64edd272cdec3bc66278105d2652ab4f8d2 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 14:25:11 -0700 Subject: [PATCH 529/752] cleanup --- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f36ba2b214a7..39f74e397f20 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -200,7 +200,8 @@ def download(self) -> None: } def process(self) -> None: - self.model = SentenceTransformer().to(self.device) + self.model = SentenceTransformer( + "sentence-transformers/all-roberta-large-v1").to(self.device) self.model.eval() self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] From 8bb52ebeb252593c332d98699a41c76cbc346e02 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 14:39:51 -0700 Subject: [PATCH 530/752] minor fix --- test/nn/models/test_g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index e39ffafad092..55fb1ab3730e 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -36,6 +36,6 @@ def test_g_retriever() -> None: assert loss is not None # test inference - out = model.inference(batch.question, batch.x, batch.edge_index, + pred = model.inference(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.edge_attr) - assert out['pred'] is not None + assert pred is not None From 5a2b9d1e3821f14ac5d280cce8201175604ca3f2 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 14:40:36 -0700 Subject: [PATCH 531/752] cleanup --- test/nn/models/test_g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index e39ffafad092..55fb1ab3730e 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -36,6 +36,6 @@ def test_g_retriever() -> None: assert loss is not None # test inference - out = model.inference(batch.question, batch.x, batch.edge_index, + pred = model.inference(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.edge_attr) - assert out['pred'] is not None + assert pred is not None From 20ef4f24edac7902983fbb9421dd15f42104f50a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:41:26 +0000 Subject: [PATCH 532/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/nn/models/test_g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 55fb1ab3730e..9529adab97bc 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -37,5 +37,5 @@ def test_g_retriever() -> None: # test inference pred = model.inference(batch.question, batch.x, batch.edge_index, - batch.batch, batch.ptr, batch.edge_attr) + batch.batch, batch.ptr, batch.edge_attr) assert pred is not None From 4f4d2e511f2431b03853aef6e929560b3f87a979 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:42:35 +0000 Subject: [PATCH 533/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/nn/models/test_g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 55fb1ab3730e..9529adab97bc 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -37,5 +37,5 @@ def test_g_retriever() -> None: # test inference pred = model.inference(batch.question, batch.x, batch.edge_index, - batch.batch, batch.ptr, batch.edge_attr) + batch.batch, batch.ptr, batch.edge_attr) assert pred is not None From 9856171bd3cca040aa0bd7b605f27622d3470de5 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 15:04:50 -0700 Subject: [PATCH 534/752] warning to let people know that CPU LLMs can be prohibitively slow --- torch_geometric/nn/nlp/llm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 53b9c68236da..7ad51f06034f 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -88,6 +88,9 @@ def __init__( self.word_embedding = self.llm.model.get_input_embeddings() if 'max_memory' not in kwargs: # Pure CPU: + print("Warning: LLM is being run w/ CPU, this may be extremely slow.") + print("If you do not have a GPU, consider getting one.") + print("If you do, you need more VRAM either by upgrading or getting another GPU.") # noqa self.llm_device = torch.device('cpu') self.autocast_context = nullcontext() else: From 081ac02b7e6e15bc06856a2411dcbb60c90de394 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 22:06:01 +0000 Subject: [PATCH 535/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/llm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 7ad51f06034f..3fd33a303795 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -88,9 +88,13 @@ def __init__( self.word_embedding = self.llm.model.get_input_embeddings() if 'max_memory' not in kwargs: # Pure CPU: - print("Warning: LLM is being run w/ CPU, this may be extremely slow.") + print( + "Warning: LLM is being run w/ CPU, this may be extremely slow." + ) print("If you do not have a GPU, consider getting one.") - print("If you do, you need more VRAM either by upgrading or getting another GPU.") # noqa + print( + "If you do, you need more VRAM either by upgrading or getting another GPU." + ) # noqa self.llm_device = torch.device('cpu') self.autocast_context = nullcontext() else: From 44f94c00c03c36b91c5a57a2b8ef4f323feea722 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 15:09:56 -0700 Subject: [PATCH 536/752] cleaning --- torch_geometric/nn/nlp/sentence_transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/nn/nlp/sentence_transformer.py b/torch_geometric/nn/nlp/sentence_transformer.py index cfb26176fcc6..e102eb037e5b 100644 --- a/torch_geometric/nn/nlp/sentence_transformer.py +++ b/torch_geometric/nn/nlp/sentence_transformer.py @@ -54,6 +54,8 @@ def encode( batch_size: Optional[int] = None, output_device: Optional[torch.device] = None, ) -> Tensor: + if len(text) == 0: + return torch.zeros((0, 1024)) batch_size = len(text) if batch_size is None else batch_size embs: List[Tensor] = [] From 72300b3c5d74ebf035ceb14fd55759ad01e40bbb Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 2 Jul 2024 15:11:03 -0700 Subject: [PATCH 537/752] clean --- torch_geometric/nn/nlp/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 3fd33a303795..549d558c3c52 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -93,8 +93,8 @@ def __init__( ) print("If you do not have a GPU, consider getting one.") print( - "If you do, you need more VRAM either by upgrading or getting another GPU." - ) # noqa + "If you do, you need more VRAM either by upgrading or getting another GPU." # noqa + ) self.llm_device = torch.device('cpu') self.autocast_context = nullcontext() else: From d1d1a59d90a3d32a65bd958f78df9e1338d94bf0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 22:12:34 +0000 Subject: [PATCH 538/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 549d558c3c52..0e8cf5b64026 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -93,7 +93,7 @@ def __init__( ) print("If you do not have a GPU, consider getting one.") print( - "If you do, you need more VRAM either by upgrading or getting another GPU." # noqa + "If you do, you need more VRAM either by upgrading or getting another GPU." # noqa ) self.llm_device = torch.device('cpu') self.autocast_context = nullcontext() From 118dc09375471dcb44ceb38fac309e939a46c544 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Wed, 3 Jul 2024 17:29:01 -0700 Subject: [PATCH 539/752] quick fix --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 07778bbeadfa..3f9939b784ca 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -316,7 +316,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, # GNN+LLM only using 32 tokens to answer. # Allow more output tokens for untrained LLM pure_llm_pred = pure_llm.inference(batch.question, batch.desc, - max_out_tokens=256) + max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] From 57da8342583f8ccc240a544388b3e8461f61a358 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 11 Jul 2024 17:53:06 -0700 Subject: [PATCH 540/752] Modularizing LLM (G-retriever) code (#9502) will merge this in when its ready to address https://github.com/pyg-team/pytorch_geometric/pull/9480#discussion_r1668461895 halfway done modularizing --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- torch_geometric/nn/models/g_retriever.py | 117 ++----------- torch_geometric/nn/nlp/llm.py | 202 ++++++++++++----------- 2 files changed, 117 insertions(+), 202 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index d68e9b8f61d8..500f5992fa1f 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -4,13 +4,7 @@ import torch.nn as nn from torch_geometric.nn.models import GAT -from torch_geometric.nn.nlp.llm import ( - EOS, - IGNORE_INDEX, - LLM, - MAX_NEW_TOKENS, - MAX_TXT_LEN, -) +from torch_geometric.nn.nlp.llm import LLM, MAX_NEW_TOKENS from torch_geometric.utils import scatter @@ -163,65 +157,17 @@ def forward( additional_text_context (List[str], optional): Additional context to give to the LLM, such as textified knowledge graphs. """ - batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa - # encode labels - labels = self.tokenizer(label, add_special_tokens=False) - # encode training specific special token - eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - - # encode graphs + num_nodes_per_graph = ptr[1:] - ptr[:-1] graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, batch) - graph_embeds = self.projector(graph_embeds) - batch_inputs_embeds = [] - batch_attention_mask = [] - batch_label_input_ids = [] - num_nodes_per_graph = ptr[1:] - ptr[:-1] - for i in range(batch_size): - # Add bos & eos token - label_input_ids = labels.input_ids[ - i][:MAX_NEW_TOKENS] + eos_tokens.input_ids - if additional_text_context is not None: - input_ids = context.input_ids[ - i][:MAX_TXT_LEN] + questions.input_ids[ - i] + eos_user_tokens.input_ids + label_input_ids - else: - input_ids = questions.input_ids[ - i] + eos_user_tokens.input_ids + label_input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - to_cat = [bos_embeds] - if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], - dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - label_input_ids = [IGNORE_INDEX - ] * (inputs_embeds.shape[0] - - len(label_input_ids)) + label_input_ids - batch_label_input_ids.append(label_input_ids) + graph_embeds = [ + (embed.unsqueeze(0) if num_nodes_per_graph[i] != 0 else None) + for i, embed in enumerate(self.projector(graph_embeds)) + ] - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat([ - pad_embeds.repeat(pad_length, 1).to(self.llm_device), - batch_inputs_embeds[i].to(self.llm_device) - ]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - batch_label_input_ids[ - i] = [IGNORE_INDEX] * pad_length + batch_label_input_ids[i] + inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa + question, additional_text_context, graph_embeds, label) - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - label_input_ids = torch.tensor(batch_label_input_ids).to( - self.llm_device) with self.llm_to_use.autocast_context: outputs = self.llm_generator( inputs_embeds=inputs_embeds, @@ -263,48 +209,15 @@ def inference( max_out_tokens (int, optional): How many tokens for the LLM to generate. (default: {32}) """ - batch_size, questions, context, eos_user_tokens, \ - bos_embeds, pad_embeds = self.llm_to_use._encode_inputs(question, additional_text_context) # noqa - # encode graphs + num_nodes_per_graph = ptr[1:] - ptr[:-1] graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, batch) - graph_embeds = self.projector(graph_embeds) - - batch_inputs_embeds = [] - batch_attention_mask = [] - num_nodes_per_graph = ptr[1:] - ptr[:-1] - for i in range(batch_size): - # Add bos & eos token - if additional_text_context is not None: - input_ids = context.input_ids[ - i][:MAX_TXT_LEN] + questions.input_ids[ - i] + eos_user_tokens.input_ids - else: - input_ids = questions.input_ids[i] + eos_user_tokens.input_ids - inputs_embeds = self.word_embedding( - torch.tensor(input_ids).to(self.llm_device)) - to_cat = [bos_embeds] - if num_nodes_per_graph[i] != 0: - to_cat.append(graph_embeds[i].unsqueeze(0)) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], - dim=0) - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.shape[0]) - - # pad inputs_embeds - max_length = max([x.shape[0] for x in batch_inputs_embeds]) - for i in range(batch_size): - pad_length = max_length - batch_inputs_embeds[i].shape[0] - batch_inputs_embeds[i] = torch.cat( - [pad_embeds.repeat(pad_length, 1), batch_inputs_embeds[i]]) - batch_attention_mask[i] = [0 - ] * pad_length + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, - dim=0).to(self.llm_device) - attention_mask = torch.tensor(batch_attention_mask).to(self.llm_device) - + graph_embeds = [ + (embed.unsqueeze(0) if num_nodes_per_graph[i] != 0 else None) + for i, embed in enumerate(self.projector(graph_embeds)) + ] + inputs_embeds, attention_mask, _ = self.llm_to_use._get_embeds( + question, additional_text_context, graph_embeds) with self.llm_to_use.autocast_context: outputs = self.llm_generator.generate( inputs_embeds=inputs_embeds, diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 53b9c68236da..8b1b8b4874c8 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -1,4 +1,3 @@ -import warnings from contextlib import nullcontext from typing import Any, Dict, List, Optional @@ -117,6 +116,102 @@ def _encode_inputs( return (batch_size, questions, context, eos_user_tokens, bos_embeds, pad_embeds) + def _label_input_ids(self, i, label, eos_tokens): + label_input_ids = label.input_ids[i][:MAX_NEW_TOKENS] + label_input_ids += eos_tokens.input_ids # Add EOS token. + return label_input_ids + + def _input_ids(self, i, context, question, eos_user_tokens): + input_ids: List[int] = [] + if context is not None: + input_ids += context.input_ids[i][:MAX_TXT_LEN] + input_ids += question.input_ids[i] + input_ids += eos_user_tokens.input_ids + return input_ids + + def _inputs_embeds(self, i, input_ids, bos_embeds, embedding=None): + inputs_embeds = self.word_embedding( + torch.tensor(input_ids, device=self.llm_device)) + + to_cat = [bos_embeds] + if embedding is not None: + to_cat.append(embedding[i]) + to_cat.append(inputs_embeds) + inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], + dim=0) + return inputs_embeds + + def _append_embeds(self, inputs_embeds, batch_inputs_embeds, + batch_attention_mask, label_input_ids=None, + batch_label_input_ids=None): + batch_inputs_embeds.append(inputs_embeds) + batch_attention_mask.append([1] * inputs_embeds.size(0)) + if label_input_ids is not None: + label_input_ids = [IGNORE_INDEX] * ( + inputs_embeds.size(0) - len(label_input_ids)) + label_input_ids + batch_label_input_ids.append(label_input_ids) + return batch_inputs_embeds, batch_attention_mask, batch_label_input_ids + + def _pad_embeds(self, pad_embeds, batch_inputs_embeds, + batch_attention_mask, batch_label_input_ids=None): + max_length = max([x.size(0) for x in batch_inputs_embeds]) + batch_size = len(batch_inputs_embeds) + for i in range(batch_size): + pad = max_length - batch_inputs_embeds[i].size(0) + batch_inputs_embeds[i] = torch.cat([ + pad_embeds.repeat(pad, 1), + batch_inputs_embeds[i], + ]) + batch_attention_mask[i] = [0] * pad + batch_attention_mask[i] + if batch_label_input_ids is not None: + batch_label_input_ids[i] = ([IGNORE_INDEX] * pad + + batch_label_input_ids[i]) + inputs_embeds = torch.stack(batch_inputs_embeds, dim=0) + attention_mask = torch.tensor(batch_attention_mask, + device=self.llm_device) + if batch_label_input_ids is not None: + label_input_ids = torch.tensor(batch_label_input_ids, + device=self.llm_device) + else: + label_input_ids = None + return inputs_embeds, attention_mask, label_input_ids + + def _get_embeds(self, question, context=None, embedding=None, answer=None): + (batch_size, question, context, eos_user_tokens, bos_embeds, + pad_embeds) = self._encode_inputs(question, context) + if answer is not None: + label = self.tokenizer(answer, add_special_tokens=False) + eos_tokens = self.tokenizer(EOS, add_special_tokens=False) + batch_label_input_ids = [] + else: + batch_label_input_ids = None + + batch_inputs_embeds = [] + batch_attention_mask = [] + if answer is not None: + batch_label_input_ids = [] + else: + batch_label_input_ids = None + for i in range(batch_size): + input_ids = self._input_ids(i, context, question, eos_user_tokens) + if answer is not None: + label_input_ids = self._label_input_ids(i, label, eos_tokens) + input_ids += label_input_ids + else: + label_input_ids = None + + inputs_embeds = self._inputs_embeds(i, input_ids, bos_embeds, + embedding) + + batch_inputs_embeds, batch_attention_mask, batch_label_input_ids = self._append_embeds( # noqa + inputs_embeds, batch_inputs_embeds, batch_attention_mask, + label_input_ids, batch_label_input_ids) + + inputs_embeds, attention_mask, label_input_ids = self._pad_embeds( + pad_embeds, batch_inputs_embeds, batch_attention_mask, + batch_label_input_ids) + return inputs_embeds, attention_mask, label_input_ids + def forward( self, question: List[str], @@ -133,65 +228,11 @@ def forward( LLM, such as textified knowledge graphs. (default: :obj:`None`) embedding (list[torch.Tensor], optional): RAG embedding tensors, *i.e.* the embedded form of :obj:`context`. Either - :obj:`context` or :obj:`rag_embeddings` should be used, not + :obj:`context` or :obj:`embedding` should be used, not both. (default: :obj:`None`) """ - if context is not None and embedding is not None: - warnings.warn("Using both 'context' and 'embedding' is a waste of " - "compute and memory") - - (batch_size, question, context, eos_user_tokens, bos_embeds, - pad_embeds) = self._encode_inputs(question, context) - - label = self.tokenizer(answer, add_special_tokens=False) - eos_tokens = self.tokenizer(EOS, add_special_tokens=False) - - batch_inputs_embeds = [] - batch_attention_mask = [] - batch_label_input_ids = [] - for i in range(batch_size): - label_input_ids = label.input_ids[i][:MAX_NEW_TOKENS] - label_input_ids += eos_tokens.input_ids # Add EOS token. - - input_ids: List[int] = [] - if context is not None: - input_ids += context.input_ids[i][:MAX_TXT_LEN] - input_ids += question.input_ids[i] - input_ids += eos_user_tokens.input_ids - input_ids += label_input_ids - - inputs_embeds = self.word_embedding( - torch.tensor(input_ids, device=self.llm_device)) - - to_cat = [bos_embeds] - if embedding is not None: - to_cat.append(embedding[i]) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) - - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.size(0)) - label_input_ids = [IGNORE_INDEX] * ( - inputs_embeds.size(0) - len(label_input_ids)) + label_input_ids - batch_label_input_ids.append(label_input_ids) - - # Pad input embeddings: - max_length = max([x.size(0) for x in batch_inputs_embeds]) - for i in range(batch_size): - pad = max_length - batch_inputs_embeds[i].size(0) - batch_inputs_embeds[i] = torch.cat([ - pad_embeds.repeat(pad, 1), - batch_inputs_embeds[i], - ]) - batch_attention_mask[i] = [0] * pad + batch_attention_mask[i] - batch_label_input_ids[i] = ([IGNORE_INDEX] * pad + - batch_label_input_ids[i]) - - inputs_embeds = torch.stack(batch_inputs_embeds, dim=0) - attention_mask = torch.tensor(batch_attention_mask, - device=self.llm_device) - label_input_ids = torch.tensor(batch_label_input_ids, - device=self.llm_device) + inputs_embeds, attention_mask, label_input_ids = self._get_embeds( + question, context, embedding, answer) with self.autocast_context: outputs = self.llm( @@ -219,52 +260,13 @@ def inference( LLM, such as textified knowledge graphs. (default: :obj:`None`) embedding (list[torch.Tensor], optional): RAG embedding tensors, *i.e.* the embedded form of :obj:`context`. Either - :obj:`context` or :obj:`rag_embeddings` should be used, not + :obj:`context` or :obj:`embedding` should be used, not both. (default: :obj:`None`) max_tokens (int, optional): How many tokens for the LLM to generate. (default: :obj:`32`) """ - if context is not None and embedding is not None: - warnings.warn("Using both 'context' and 'embedding' is a waste of " - "compute and memory") - - (batch_size, question, context, eos_user_tokens, bos_embeds, - pad_embeds) = self._encode_inputs(question, context) - - batch_inputs_embeds = [] - batch_attention_mask = [] - for i in range(batch_size): - input_ids: List[int] = [] - if context is not None: - input_ids = context.input_ids[i][:MAX_TXT_LEN] - input_ids += question.input_ids[i] - input_ids += eos_user_tokens.input_ids - - inputs_embeds = self.word_embedding( - torch.tensor(input_ids, device=self.llm_device)) - - to_cat = [bos_embeds] - if embedding is not None: - to_cat.append(embedding[i]) - to_cat.append(inputs_embeds) - inputs_embeds = torch.cat(to_cat, dim=0) - - batch_inputs_embeds.append(inputs_embeds) - batch_attention_mask.append([1] * inputs_embeds.size(0)) - - # Pad input embeddings: - max_length = max([x.size(0) for x in batch_inputs_embeds]) - for i in range(batch_size): - pad = max_length - batch_inputs_embeds[i].size(0) - batch_inputs_embeds[i] = torch.cat([ - pad_embeds.repeat(pad, 1), - batch_inputs_embeds[i], - ]) - batch_attention_mask[i] = [0] * pad + batch_attention_mask[i] - - inputs_embeds = torch.stack(batch_inputs_embeds, dim=0) - attention_mask = torch.tensor(batch_attention_mask, - device=self.llm_device) + inputs_embeds, attention_mask, _ = self._get_embeds( + question, context, embedding) bos_token = self.tokenizer( BOS, From 12c71c3c2afc1b640a1106549a1861f4f1f4d115 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 25 Jul 2024 09:12:40 -0700 Subject: [PATCH 541/752] fix for bug --- torch_geometric/nn/models/g_retriever.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 500f5992fa1f..81ccb19ea035 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -159,11 +159,13 @@ def forward( """ num_nodes_per_graph = ptr[1:] - ptr[:-1] graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, - batch) - graph_embeds = [ - (embed.unsqueeze(0) if num_nodes_per_graph[i] != 0 else None) - for i, embed in enumerate(self.projector(graph_embeds)) - ] + projected_graph_embeds = self.projector(graph_embeds) + graph_embeds = [] + for i, num_nodes_in_graph_i in enumerate(num_nodes_per_graph): + if num_nodes_in_graph == 0: + graph_embeds.append(None) + else: + graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa question, additional_text_context, graph_embeds, label) From 1909e7d34a6cd2ccab0cd35a2bed2fc285548609 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 25 Jul 2024 09:23:38 -0700 Subject: [PATCH 542/752] accidentally deleted a line, putting it back --- torch_geometric/nn/models/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 81ccb19ea035..00f0560ef7b7 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -159,6 +159,7 @@ def forward( """ num_nodes_per_graph = ptr[1:] - ptr[:-1] graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, + batch) projected_graph_embeds = self.projector(graph_embeds) graph_embeds = [] for i, num_nodes_in_graph_i in enumerate(num_nodes_per_graph): @@ -167,7 +168,7 @@ def forward( else: graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) - inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa + inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: From 3a336fc20fa6b06aa59d5272cc6f2891cc37fe9d Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 25 Jul 2024 09:28:19 -0700 Subject: [PATCH 543/752] cleaning --- torch_geometric/nn/models/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 00f0560ef7b7..39c94878e57c 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -163,12 +163,12 @@ def forward( projected_graph_embeds = self.projector(graph_embeds) graph_embeds = [] for i, num_nodes_in_graph_i in enumerate(num_nodes_per_graph): - if num_nodes_in_graph == 0: + if num_nodes_in_graph_i == 0: graph_embeds.append(None) else: graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) - inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( + inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: From 78643c974cda5973d34dcc020b1f6cad070a190d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:30:09 +0000 Subject: [PATCH 544/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 39c94878e57c..21a1f9123d20 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -168,7 +168,7 @@ def forward( else: graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) - inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa + inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: From 952c539f0fb14a389554f9b7e2dd975f6c91b620 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:32:28 +0000 Subject: [PATCH 545/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 39c94878e57c..21a1f9123d20 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -168,7 +168,7 @@ def forward( else: graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) - inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa + inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: From d407cbd289b19a90f086f66f28aecfb8484e622b Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 25 Jul 2024 09:41:45 -0700 Subject: [PATCH 546/752] last minor fix --- torch_geometric/nn/nlp/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 8b1b8b4874c8..8f019d4dca98 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -134,7 +134,7 @@ def _inputs_embeds(self, i, input_ids, bos_embeds, embedding=None): torch.tensor(input_ids, device=self.llm_device)) to_cat = [bos_embeds] - if embedding is not None: + if embedding[i] is not None: to_cat.append(embedding[i]) to_cat.append(inputs_embeds) inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], From dd651be4944b0ff940c17e7e800604dedecaa044 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 25 Jul 2024 09:56:58 -0700 Subject: [PATCH 547/752] hopefuly final minor fix from my rework --- torch_geometric/nn/nlp/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 8f019d4dca98..d5b3e9a28096 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -134,7 +134,7 @@ def _inputs_embeds(self, i, input_ids, bos_embeds, embedding=None): torch.tensor(input_ids, device=self.llm_device)) to_cat = [bos_embeds] - if embedding[i] is not None: + if embedding is not None and embedding[i] is not None: to_cat.append(embedding[i]) to_cat.append(inputs_embeds) inputs_embeds = torch.cat([i.to(self.llm_device) for i in to_cat], From 6d54d5a99ff3997b1dbd2e2fe5b079480ed4e2c4 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Fri, 26 Jul 2024 15:10:34 +0000 Subject: [PATCH 548/752] fix docs, model selection --- test/nn/models/test_g_retriever.py | 2 +- torch_geometric/nn/models/g_retriever.py | 52 +++++++++++++----------- torch_geometric/nn/nlp/llm.py | 11 +---- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 9529adab97bc..01be4313da47 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -6,7 +6,7 @@ @onlyFullTest -@withPackage('transformers', 'accelerate') +@withPackage('transformers', 'sentencepiece', 'accelerate') def test_g_retriever() -> None: batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 21a1f9123d20..268d5127dd18 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -11,16 +11,10 @@ class GRetriever(nn.Module): r"""This GNN+LLM implementation is based on G-retriever. Original Paper: `_. - See `examples/llm_plus_gnn/g_retriever.py` for example usage. Args: llm_to_use (str): A string representing the huggingface model you - want to use. This module has been tested for 'llama2' and 'gemma'. - Other huggingface transformer models should work if you pass the - correct name, see huggingface.co for details. If any issues occur - please file an issue on - https://github.com/pyg-team/pytorch_geometric - and assign to puririshi98. (default: :obj:'llama2') + want to use. (default: :obj:`"meta-llama/Llama-2-7b-chat-hf"`) llm_use_lora (bool): use LORA from peft for training the LLM. see https://huggingface.co/docs/peft/en/index for details. llm_dtype (torch.dtype): The dtype to use for the LLM. @@ -31,18 +25,31 @@ class GRetriever(nn.Module): available GPU memory of your GPUs (default :obj:`7`) gnn_to_use (BasicGNN): Please pass a valid model that extends torch_geometric.nn.models.basic_gnn.BasicGNN. (default: :obj:`GAT`) - gnn_in_channels (int): (default: 1024) - gnn_hidden_channels (int): (default: 1024) - gnn_out_channels (int): (default: 1024) - num_gnn_layers (int): (default: 4) + gnn_in_channels (int): (default: :obj:`1024`) + gnn_hidden_channels (int): (default: obj:`1024`) + gnn_out_channels (int): (default: :obj:`1024`) + num_gnn_layers (int): (default: :obj:`4`) num_gnn_heads (int): Number of heads to use for BasicGNNs with the - `heads` kwarg. (default: 4) - mlp_hidden_dim (int): (default: 2048) - mlp_out_dim (int): (default: 4096) + :obj:`heads` kwarg. (default: :obj:`4`) + mlp_hidden_dim (int): (default: :obj:`2048`) + mlp_out_dim (int): (default: :obj:`4096`) + + .. warning:: + This module has been tested with the following Hugging Face models + + - :obj:`llm_to_use="meta-llama/Llama-2-7b-chat-hf"` + - :obj:`llm_to_use="google/gemma-7b"` + + and may not work with other models. See other models at `Hugging Face + Models `_ and let us know if you + encounter any issue by submitting a GitHub issue. + + .. note:: + See `examples/llm_plus_gnn/g_retriever.py` for example usage. """ def __init__( self, - llm_to_use='llama2-7b', + llm_to_use='meta-llama/Llama-2-7b-chat-hf', llm_use_lora: bool = False, llm_dtype=torch.bfloat16, num_llm_params: int = 7, @@ -56,12 +63,7 @@ def __init__( mlp_out_dim: int = 4096, ) -> None: super().__init__() - if 'llama2-7b' in llm_to_use.lower(): - self.llm_to_use = LLM('llama2-7b', num_llm_params, llm_dtype) - elif 'gemma' in llm_to_use.lower(): - self.llm_to_use = LLM('gemma', num_llm_params, llm_dtype) - else: - self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) + self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) self.llm_generator = self.llm_to_use.llm self.llm_dtype = llm_dtype if llm_use_lora: @@ -70,7 +72,6 @@ def __init__( get_peft_model, prepare_model_for_kbit_training, ) - print("Training our LLM with LORA!") self.llm_generator = prepare_model_for_kbit_training( self.llm_generator) lora_r: int = 8 @@ -125,7 +126,6 @@ def encode_graphs(self, node_feat, edge_index, edge_attr, batch): edge_attr = edge_attr.to(self.llm_device) n_embeds = self.graph_encoder(x, edge_index.long(), edge_attr) batch = batch.to(self.llm_device) - # mean pooling g_embeds = scatter(n_embeds, batch, dim=0, reduce='mean') return g_embeds @@ -168,7 +168,11 @@ def forward( else: graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) - inputs_embeds, attention_mask, label_input_ids = self.llm_to_use._get_embeds( # noqa + ( + inputs_embeds, + attention_mask, + label_input_ids, + ) = self.llm_to_use._get_embeds( # noqa question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index d5b3e9a28096..c9b044277c9c 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -62,21 +62,12 @@ def __init__( from transformers import AutoModelForCausalLM, AutoTokenizer - if model_name == 'llama2-7b': - pretty_model_name = 'LLAMA2' - model_name = 'meta-llama/Llama-2-7b-chat-hf' - elif model_name == 'gemma': - pretty_model_name = 'GEMMA' - model_name = 'google/gemma-7b' - else: - pretty_model_name = model_name - # A rough heuristic on GPU memory requirements, e.g., we found that # LLAMA2 (7B parameters) fits on a 85GB GPU. required_memory = 85 * num_params / 7 kwargs = get_llm_kwargs(required_memory, dtype) - print(f"Setting up '{pretty_model_name}' with configuration: {kwargs}") + print(f"Setting up '{model_name}' with configuration: {kwargs}") self.tokenizer = AutoTokenizer.from_pretrained( model_name, use_fast=False, From 31d8f2beded5710c36374b778563567c233fd8e1 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Jul 2024 14:27:04 -0700 Subject: [PATCH 549/752] Update g_retriever.py --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3f9939b784ca..19df454ade18 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -162,7 +162,7 @@ def adjust_learning_rate(param_group, LR, epoch): mlp_out_dim=2048, ) else: - model = GRetriever(gnn_hidden_channels=hidden_channels, + model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) if num_gnn_layers is not None: model_save_name = "gnn_llm" From dc44a7ded80d053675b137bedfb729312f39fb21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:28:22 +0000 Subject: [PATCH 550/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 19df454ade18..126508fd3e9c 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -162,7 +162,8 @@ def adjust_learning_rate(param_group, LR, epoch): mlp_out_dim=2048, ) else: - model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", gnn_hidden_channels=hidden_channels, + model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", + gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) if num_gnn_layers is not None: model_save_name = "gnn_llm" From 63340d8bc00370718d552952338ad354c2129d48 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Jul 2024 14:41:05 -0700 Subject: [PATCH 551/752] Update g_retriever.py --- examples/llm_plus_gnn/g_retriever.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 126508fd3e9c..882df2fd5765 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -180,11 +180,6 @@ def adjust_learning_rate(param_group, LR, epoch): }, ], betas=(0.9, 0.95)) grad_steps = 2 - if model is None: - trainable_params, all_param = model.print_trainable_params() - print(f"trainable params: {trainable_params} || \ - all params: {all_param} || \ - trainable%: {100 * trainable_params / all_param}") best_val_loss = float('inf') # Step 4 Training From 21224b1474f3702673e5a1280cf3337101c1028c Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Jul 2024 14:42:41 -0700 Subject: [PATCH 552/752] Update torch_geometric/nn/models/g_retriever.py Co-authored-by: Akihiro Nitta --- torch_geometric/nn/models/g_retriever.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 268d5127dd18..aba7f651a751 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -235,14 +235,3 @@ def inference( ) return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - def print_trainable_params(self) -> None: - trainable_params = 0 - all_param = 0 - for _, param in self.named_parameters(): - num_params = param.numel() - - all_param += num_params - if param.requires_grad: - trainable_params += num_params - - return trainable_params, all_param From 98523d1b172c9173197d43a70aa1c754a420039b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:50:17 +0000 Subject: [PATCH 553/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index aba7f651a751..6dd204a576f7 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -234,4 +234,3 @@ def inference( use_cache=True # IMPORTANT! ) return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - From 3f58f90b6045871da722abc335e86d2ebeaca50b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:52:34 +0000 Subject: [PATCH 554/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index aba7f651a751..6dd204a576f7 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -234,4 +234,3 @@ def inference( use_cache=True # IMPORTANT! ) return self.tokenizer.batch_decode(outputs, skip_special_tokens=True) - From 4afb5cd9b4f670a604bc41b9d5db76d205c69629 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Jul 2024 15:17:37 -0700 Subject: [PATCH 555/752] noqa for bare except --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 6dd204a576f7..a318addb0e51 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -101,7 +101,7 @@ def __init__( heads=num_gnn_heads, norm='batch_norm', ).to(self.llm_device) - except: # noqa + except: # noqa: E722 # to handle gnns that do not have `heads` param self.graph_encoder = gnn_to_use( in_channels=gnn_in_channels, From 7238400b40df67aad87060835cbc9ca33c539619 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Mon, 29 Jul 2024 15:27:23 -0700 Subject: [PATCH 556/752] removing unneeded noqa --- torch_geometric/nn/models/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index a318addb0e51..01abaaef3e04 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -172,7 +172,7 @@ def forward( inputs_embeds, attention_mask, label_input_ids, - ) = self.llm_to_use._get_embeds( # noqa + ) = self.llm_to_use._get_embeds( question, additional_text_context, graph_embeds, label) with self.llm_to_use.autocast_context: From c4d32eb85b24e1d8152e00325680afaf85c6f3fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 22:28:31 +0000 Subject: [PATCH 557/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/models/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 01abaaef3e04..573434fc00fa 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -172,8 +172,8 @@ def forward( inputs_embeds, attention_mask, label_input_ids, - ) = self.llm_to_use._get_embeds( - question, additional_text_context, graph_embeds, label) + ) = self.llm_to_use._get_embeds(question, additional_text_context, + graph_embeds, label) with self.llm_to_use.autocast_context: outputs = self.llm_generator( From 09de7d0ecd07663e1b8637fb7a30fa38c0bc71c8 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Thu, 1 Aug 2024 16:55:15 -0700 Subject: [PATCH 558/752] fix for recent changes intro-ing a typo --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 882df2fd5765..af148ba5b4fa 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -279,7 +279,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, num_params=1, ) else: - pure_llm = LLM(model_name="llama2-7b", num_params=7) + pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", num_params=7) if path.exists("demo_save_dict.pt"): print("Saved outputs for the first step of the demo found.") print("Would you like to redo?") From 625b93ee5be49fffc9de163743de72796227cb00 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 23:56:28 +0000 Subject: [PATCH 559/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index af148ba5b4fa..e3f5a12df4e9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -279,7 +279,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, num_params=1, ) else: - pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", num_params=7) + pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", + num_params=7) if path.exists("demo_save_dict.pt"): print("Saved outputs for the first step of the demo found.") print("Would you like to redo?") From 89efa75b9b99376c6e3ed2c20849d5d90ca97e43 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 12:10:08 -0700 Subject: [PATCH 560/752] Update test_g_retriever.py --- test/nn/models/test_g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 01be4313da47..ef503da909c8 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -6,7 +6,7 @@ @onlyFullTest -@withPackage('transformers', 'sentencepiece', 'accelerate') +@withPackage('transformers', 'sentencepiece') def test_g_retriever() -> None: batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] From 686077030d0f928ce5793f18b1c634d86d5be9fb Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 12:11:38 -0700 Subject: [PATCH 561/752] removing unneeded requirement (only needed for larger multigpu llms) --- test/nn/nlp/test_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index 03fa29622f5b..4a7a885c0c4f 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -6,7 +6,7 @@ @onlyFullTest -@withPackage('transformers', 'accelerate') +@withPackage('transformers') def test_llm() -> None: question = ["Is PyG the best open-source GNN library?"] answer = ["yes!"] From 54f653ab0a027b0e807e031fcedfd1c0cef3b7f6 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 12:18:14 -0700 Subject: [PATCH 562/752] undoing --- test/nn/nlp/test_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/nlp/test_llm.py b/test/nn/nlp/test_llm.py index 4a7a885c0c4f..03fa29622f5b 100644 --- a/test/nn/nlp/test_llm.py +++ b/test/nn/nlp/test_llm.py @@ -6,7 +6,7 @@ @onlyFullTest -@withPackage('transformers') +@withPackage('transformers', 'accelerate') def test_llm() -> None: question = ["Is PyG the best open-source GNN library?"] answer = ["yes!"] From d794dc2f7c97f2c674a03ee11167448ded237c88 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 12:18:49 -0700 Subject: [PATCH 563/752] undoing --- test/nn/models/test_g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index ef503da909c8..01be4313da47 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -6,7 +6,7 @@ @onlyFullTest -@withPackage('transformers', 'sentencepiece') +@withPackage('transformers', 'sentencepiece', 'accelerate') def test_g_retriever() -> None: batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] From 068bded5a95d0e64c96dc5ee03f125a954cc520f Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:44:59 -0700 Subject: [PATCH 564/752] Update conftest.py --- test/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index d4499d477abf..df585dfa70d0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -42,6 +42,8 @@ def load_dataset(root: str, name: str, *args, **kwargs) -> Dataset: if name.lower() in ['hetero']: from torch_geometric.testing import FakeHeteroDataset return FakeHeteroDataset(*args, **kwargs) + if name.lower() == 'webqspdataset' + return WebQSPDataset() raise ValueError(f"Cannot load dataset with name '{name}'") From 834a8084688daa278ee16f287ed9053dc205ce5a Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:46:59 -0700 Subject: [PATCH 565/752] Update conftest.py --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index df585dfa70d0..eef87a1d6ee2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -42,7 +42,7 @@ def load_dataset(root: str, name: str, *args, **kwargs) -> Dataset: if name.lower() in ['hetero']: from torch_geometric.testing import FakeHeteroDataset return FakeHeteroDataset(*args, **kwargs) - if name.lower() == 'webqspdataset' + if name.lower() == 'webqspdataset': return WebQSPDataset() raise ValueError(f"Cannot load dataset with name '{name}'") From fb532c05c0c023a2099dd4a04936076f2469ac45 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:48:42 -0700 Subject: [PATCH 566/752] Update test_web_qsp_dataset.py --- test/datasets/test_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 9b484da78bda..395b406c228f 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,5 +1,5 @@ from torch_geometric.testing import onlyFullTest, onlyOnline - +from torch_geometric.datasets import WebQSPDataset @onlyOnline @onlyFullTest From 92d80cf8e465930c04ac2d3b922c1cbd0243c151 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 21:49:56 +0000 Subject: [PATCH 567/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/datasets/test_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 395b406c228f..9b484da78bda 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,5 +1,5 @@ from torch_geometric.testing import onlyFullTest, onlyOnline -from torch_geometric.datasets import WebQSPDataset + @onlyOnline @onlyFullTest From 7a585b52ea09453dc61d61713b31a909bc5b6ff1 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:50:51 -0700 Subject: [PATCH 568/752] Update test_web_qsp_dataset.py --- test/datasets/test_web_qsp_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 9b484da78bda..220d2319cad0 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,9 +1,10 @@ from torch_geometric.testing import onlyFullTest, onlyOnline +from torch_geometric.datasets import WebQSPDataset @onlyOnline @onlyFullTest def test_web_qsp_dataset(get_dataset): - dataset = get_dataset(name='WebQSPDataset') + dataset = WebQSPDataset() assert len(dataset) == 4700 assert str(dataset) == "WebQSPDataset(4700)" From dde79cf78b973bd234790a37bd3a603dbda8f178 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:51:16 -0700 Subject: [PATCH 569/752] Update conftest.py --- test/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index eef87a1d6ee2..d4499d477abf 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -42,8 +42,6 @@ def load_dataset(root: str, name: str, *args, **kwargs) -> Dataset: if name.lower() in ['hetero']: from torch_geometric.testing import FakeHeteroDataset return FakeHeteroDataset(*args, **kwargs) - if name.lower() == 'webqspdataset': - return WebQSPDataset() raise ValueError(f"Cannot load dataset with name '{name}'") From e849fbae399dcb1717f2264d215d8dbfe2b18633 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Fri, 16 Aug 2024 14:51:35 -0700 Subject: [PATCH 570/752] Update test_web_qsp_dataset.py --- test/datasets/test_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 220d2319cad0..895729399321 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -4,7 +4,7 @@ @onlyOnline @onlyFullTest -def test_web_qsp_dataset(get_dataset): +def test_web_qsp_dataset(): dataset = WebQSPDataset() assert len(dataset) == 4700 assert str(dataset) == "WebQSPDataset(4700)" From 81582553255ab3eecab51cfa312c2091103e59fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 21:52:24 +0000 Subject: [PATCH 571/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/datasets/test_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 895729399321..4079f09b6484 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,5 +1,5 @@ -from torch_geometric.testing import onlyFullTest, onlyOnline from torch_geometric.datasets import WebQSPDataset +from torch_geometric.testing import onlyFullTest, onlyOnline @onlyOnline From 095123a1f76a1a30fe5f33ffe459a88f8e31cbc6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:45:32 +0000 Subject: [PATCH 572/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 713c3acc7de5..17a51ff6d252 100644 --- a/README.md +++ b/README.md @@ -383,9 +383,9 @@ where `${CUDA}` should be replaced by either `cpu`, `cu118`, `cu121`, or `cu124` | | `cpu` | `cu118` | `cu121` | `cu124` | | ----------- | ----- | ------- | ------- | ------- | -| **Linux** | ✅ | ✅ | ✅ | ✅ | -| **Windows** | ✅ | ✅ | ✅ | ✅ | -| **macOS** | ✅ | | | | +| **Linux** | ✅ | ✅ | ✅ | ✅ | +| **Windows** | ✅ | ✅ | ✅ | ✅ | +| **macOS** | ✅ | | | | #### PyTorch 2.3 @@ -399,9 +399,9 @@ where `${CUDA}` should be replaced by either `cpu`, `cu118`, or `cu121` dependin | | `cpu` | `cu118` | `cu121` | | ----------- | ----- | ------- | ------- | -| **Linux** | ✅ | ✅ | ✅ | -| **Windows** | ✅ | ✅ | ✅ | -| **macOS** | ✅ | | | +| **Linux** | ✅ | ✅ | ✅ | +| **Windows** | ✅ | ✅ | ✅ | +| **macOS** | ✅ | | | **Note:** Binaries of older versions are also provided for PyTorch 1.4.0, PyTorch 1.5.0, PyTorch 1.6.0, PyTorch 1.7.0/1.7.1, PyTorch 1.8.0/1.8.1, PyTorch 1.9.0, PyTorch 1.10.0/1.10.1/1.10.2, PyTorch 1.11.0, PyTorch 1.12.0/1.12.1, PyTorch 1.13.0/1.13.1, PyTorch 2.0.0/2.0.1, PyTorch 2.1.0/2.1.1/2.1.2, and PyTorch 2.2.0/2.2.1/2.2.2 (following the same procedure). **For older versions, you might need to explicitly specify the latest supported version number** or install via `pip install --no-index` in order to prevent a manual installation from source. From f460f28530bacc7f687e77b9fe3533848ff0f0f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:48:00 +0000 Subject: [PATCH 573/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 713c3acc7de5..17a51ff6d252 100644 --- a/README.md +++ b/README.md @@ -383,9 +383,9 @@ where `${CUDA}` should be replaced by either `cpu`, `cu118`, `cu121`, or `cu124` | | `cpu` | `cu118` | `cu121` | `cu124` | | ----------- | ----- | ------- | ------- | ------- | -| **Linux** | ✅ | ✅ | ✅ | ✅ | -| **Windows** | ✅ | ✅ | ✅ | ✅ | -| **macOS** | ✅ | | | | +| **Linux** | ✅ | ✅ | ✅ | ✅ | +| **Windows** | ✅ | ✅ | ✅ | ✅ | +| **macOS** | ✅ | | | | #### PyTorch 2.3 @@ -399,9 +399,9 @@ where `${CUDA}` should be replaced by either `cpu`, `cu118`, or `cu121` dependin | | `cpu` | `cu118` | `cu121` | | ----------- | ----- | ------- | ------- | -| **Linux** | ✅ | ✅ | ✅ | -| **Windows** | ✅ | ✅ | ✅ | -| **macOS** | ✅ | | | +| **Linux** | ✅ | ✅ | ✅ | +| **Windows** | ✅ | ✅ | ✅ | +| **macOS** | ✅ | | | **Note:** Binaries of older versions are also provided for PyTorch 1.4.0, PyTorch 1.5.0, PyTorch 1.6.0, PyTorch 1.7.0/1.7.1, PyTorch 1.8.0/1.8.1, PyTorch 1.9.0, PyTorch 1.10.0/1.10.1/1.10.2, PyTorch 1.11.0, PyTorch 1.12.0/1.12.1, PyTorch 1.13.0/1.13.1, PyTorch 2.0.0/2.0.1, PyTorch 2.1.0/2.1.1/2.1.2, and PyTorch 2.2.0/2.2.1/2.2.2 (following the same procedure). **For older versions, you might need to explicitly specify the latest supported version number** or install via `pip install --no-index` in order to prevent a manual installation from source. From 93f268e127d6e112df499ca36f654d0dbf65327e Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Aug 2024 13:06:33 -0700 Subject: [PATCH 574/752] adding typing to helper functions --- torch_geometric/nn/nlp/llm.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index c9b044277c9c..617ee188873b 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -3,6 +3,7 @@ import torch from torch import Tensor +from transformers.tokenization_utils_base import BatchEncoding BOS = '[INST]' EOS_USER = '[/INST]' @@ -88,7 +89,7 @@ def _encode_inputs( self, question: List[str], context: Optional[List[str]] = None, - ) -> None: + ) -> tuple: batch_size = len(question) questions = self.tokenizer(question, add_special_tokens=False) if context is not None: @@ -107,12 +108,12 @@ def _encode_inputs( return (batch_size, questions, context, eos_user_tokens, bos_embeds, pad_embeds) - def _label_input_ids(self, i, label, eos_tokens): + def _label_input_ids(self, i:int, label:BatchEncoding, eos_tokens:BatchEncoding) -> List[int]: label_input_ids = label.input_ids[i][:MAX_NEW_TOKENS] label_input_ids += eos_tokens.input_ids # Add EOS token. return label_input_ids - def _input_ids(self, i, context, question, eos_user_tokens): + def _input_ids(self, i:int, context:BatchEncoding, question:BatchEncoding, eos_user_tokens:BatchEncoding) -> List[int]: input_ids: List[int] = [] if context is not None: input_ids += context.input_ids[i][:MAX_TXT_LEN] @@ -120,7 +121,7 @@ def _input_ids(self, i, context, question, eos_user_tokens): input_ids += eos_user_tokens.input_ids return input_ids - def _inputs_embeds(self, i, input_ids, bos_embeds, embedding=None): + def _inputs_embeds(self, i:int, input_ids: List[int], bos_embeds: Tensor, embedding:List[Tensor] = None) -> Tensor: inputs_embeds = self.word_embedding( torch.tensor(input_ids, device=self.llm_device)) @@ -132,9 +133,9 @@ def _inputs_embeds(self, i, input_ids, bos_embeds, embedding=None): dim=0) return inputs_embeds - def _append_embeds(self, inputs_embeds, batch_inputs_embeds, - batch_attention_mask, label_input_ids=None, - batch_label_input_ids=None): + def _append_embeds(self, inputs_embeds: Tensor, batch_inputs_embeds:List[Tensor], + batch_attention_mask: List[List[int]], label_input_ids: List[int]=None, + batch_label_input_ids: List[List[int]]=None) -> tuple: batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.size(0)) if label_input_ids is not None: @@ -143,8 +144,8 @@ def _append_embeds(self, inputs_embeds, batch_inputs_embeds, batch_label_input_ids.append(label_input_ids) return batch_inputs_embeds, batch_attention_mask, batch_label_input_ids - def _pad_embeds(self, pad_embeds, batch_inputs_embeds, - batch_attention_mask, batch_label_input_ids=None): + def _pad_embeds(self, pad_embeds:Tensor, batch_inputs_embeds:List[Tensor], + batch_attention_mask:List[List[int]], batch_label_input_ids:List[List[int]]=None) -> tuple: max_length = max([x.size(0) for x in batch_inputs_embeds]) batch_size = len(batch_inputs_embeds) for i in range(batch_size): @@ -167,7 +168,7 @@ def _pad_embeds(self, pad_embeds, batch_inputs_embeds, label_input_ids = None return inputs_embeds, attention_mask, label_input_ids - def _get_embeds(self, question, context=None, embedding=None, answer=None): + def _get_embeds(self, question:List[str], context:List[str]=None, embedding:List[Tensor]=None, answer:List[str]=None) -> tuple: (batch_size, question, context, eos_user_tokens, bos_embeds, pad_embeds) = self._encode_inputs(question, context) if answer is not None: From 96d719fd40b537b2268b64bc560232cd8aa8272f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:08:24 +0000 Subject: [PATCH 575/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/llm.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 617ee188873b..46883bca30d1 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -108,12 +108,15 @@ def _encode_inputs( return (batch_size, questions, context, eos_user_tokens, bos_embeds, pad_embeds) - def _label_input_ids(self, i:int, label:BatchEncoding, eos_tokens:BatchEncoding) -> List[int]: + def _label_input_ids(self, i: int, label: BatchEncoding, + eos_tokens: BatchEncoding) -> List[int]: label_input_ids = label.input_ids[i][:MAX_NEW_TOKENS] label_input_ids += eos_tokens.input_ids # Add EOS token. return label_input_ids - def _input_ids(self, i:int, context:BatchEncoding, question:BatchEncoding, eos_user_tokens:BatchEncoding) -> List[int]: + def _input_ids(self, i: int, context: BatchEncoding, + question: BatchEncoding, + eos_user_tokens: BatchEncoding) -> List[int]: input_ids: List[int] = [] if context is not None: input_ids += context.input_ids[i][:MAX_TXT_LEN] @@ -121,7 +124,8 @@ def _input_ids(self, i:int, context:BatchEncoding, question:BatchEncoding, eos_u input_ids += eos_user_tokens.input_ids return input_ids - def _inputs_embeds(self, i:int, input_ids: List[int], bos_embeds: Tensor, embedding:List[Tensor] = None) -> Tensor: + def _inputs_embeds(self, i: int, input_ids: List[int], bos_embeds: Tensor, + embedding: List[Tensor] = None) -> Tensor: inputs_embeds = self.word_embedding( torch.tensor(input_ids, device=self.llm_device)) @@ -133,9 +137,11 @@ def _inputs_embeds(self, i:int, input_ids: List[int], bos_embeds: Tensor, embedd dim=0) return inputs_embeds - def _append_embeds(self, inputs_embeds: Tensor, batch_inputs_embeds:List[Tensor], - batch_attention_mask: List[List[int]], label_input_ids: List[int]=None, - batch_label_input_ids: List[List[int]]=None) -> tuple: + def _append_embeds(self, inputs_embeds: Tensor, + batch_inputs_embeds: List[Tensor], + batch_attention_mask: List[List[int]], + label_input_ids: List[int] = None, + batch_label_input_ids: List[List[int]] = None) -> tuple: batch_inputs_embeds.append(inputs_embeds) batch_attention_mask.append([1] * inputs_embeds.size(0)) if label_input_ids is not None: @@ -144,8 +150,10 @@ def _append_embeds(self, inputs_embeds: Tensor, batch_inputs_embeds:List[Tensor] batch_label_input_ids.append(label_input_ids) return batch_inputs_embeds, batch_attention_mask, batch_label_input_ids - def _pad_embeds(self, pad_embeds:Tensor, batch_inputs_embeds:List[Tensor], - batch_attention_mask:List[List[int]], batch_label_input_ids:List[List[int]]=None) -> tuple: + def _pad_embeds(self, pad_embeds: Tensor, + batch_inputs_embeds: List[Tensor], + batch_attention_mask: List[List[int]], + batch_label_input_ids: List[List[int]] = None) -> tuple: max_length = max([x.size(0) for x in batch_inputs_embeds]) batch_size = len(batch_inputs_embeds) for i in range(batch_size): @@ -168,7 +176,9 @@ def _pad_embeds(self, pad_embeds:Tensor, batch_inputs_embeds:List[Tensor], label_input_ids = None return inputs_embeds, attention_mask, label_input_ids - def _get_embeds(self, question:List[str], context:List[str]=None, embedding:List[Tensor]=None, answer:List[str]=None) -> tuple: + def _get_embeds(self, question: List[str], context: List[str] = None, + embedding: List[Tensor] = None, + answer: List[str] = None) -> tuple: (batch_size, question, context, eos_user_tokens, bos_embeds, pad_embeds) = self._encode_inputs(question, context) if answer is not None: From c2ccb2ea4b89f9b0f57f00b7fb832709503ce958 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Aug 2024 13:13:56 -0700 Subject: [PATCH 576/752] fix for typing CI checker --- torch_geometric/nn/nlp/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 46883bca30d1..316f6dc0e999 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -3,7 +3,10 @@ import torch from torch import Tensor -from transformers.tokenization_utils_base import BatchEncoding +try: + from transformers.tokenization_utils_base import BatchEncoding +except: + BatchEncoding = Dict BOS = '[INST]' EOS_USER = '[/INST]' From 89a1fc3b23488636410dcf1f52e39a5c0bbc0071 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:16:39 +0000 Subject: [PATCH 577/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/nn/nlp/llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 316f6dc0e999..80d10d132534 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -3,6 +3,7 @@ import torch from torch import Tensor + try: from transformers.tokenization_utils_base import BatchEncoding except: From c9170e2b5a6319c2a6ebbe2dbd6364232cf11204 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Aug 2024 13:25:05 -0700 Subject: [PATCH 578/752] precommit fix --- torch_geometric/nn/nlp/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 80d10d132534..e585dbeb96ce 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -6,7 +6,7 @@ try: from transformers.tokenization_utils_base import BatchEncoding -except: +except ImportError: BatchEncoding = Dict BOS = '[INST]' From a1f5debd8652de82c3b1fc517c0909aaa0fe1314 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Aug 2024 13:37:10 -0700 Subject: [PATCH 579/752] doc string for hallucination eval --- examples/llm_plus_gnn/g_retriever.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index e3f5a12df4e9..ec2cfa2fe2b6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -24,7 +24,12 @@ from torch_geometric.nn.nlp import LLM -def detect_hallucinate(pred, label): +def detect_hallucinate(pred: str, label: str): + r""" + This is an approximation for the unsolved task of detecting hallucinations. + We define a hallucination as an output that contains no instances of + acceptable label. + """ try: split_pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(split_pred[0], label)) > 0 From 4ba58b9cfb75582330ae64a9bb67573d220c68cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:38:22 +0000 Subject: [PATCH 580/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/g_retriever.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ec2cfa2fe2b6..8a3a848f7fb7 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -25,8 +25,7 @@ def detect_hallucinate(pred: str, label: str): - r""" - This is an approximation for the unsolved task of detecting hallucinations. + r"""This is an approximation for the unsolved task of detecting hallucinations. We define a hallucination as an output that contains no instances of acceptable label. """ From 67435fba34331360d76c4ad9861d3e87917aa2e7 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 27 Aug 2024 13:52:03 -0700 Subject: [PATCH 581/752] Update g_retriever.py --- examples/llm_plus_gnn/g_retriever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 8a3a848f7fb7..7bc869c5e776 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -25,7 +25,7 @@ def detect_hallucinate(pred: str, label: str): - r"""This is an approximation for the unsolved task of detecting hallucinations. + r"""An approximation for the unsolved task of detecting hallucinations. We define a hallucination as an output that contains no instances of acceptable label. """ From a65a15e9bcd2796e2984a88cd164a8cda22f51a8 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 24 May 2024 22:50:31 -0700 Subject: [PATCH 582/752] dataloading whole graph --- examples/llm_plus_gnn/large_graph_indexer.py | 109 + .../llm_plus_gnn/test_eval_gretriever.ipynb | 2011 +++++++++++++++++ torch_geometric/datasets/web_qsp_dataset.py | 12 +- 3 files changed, 2128 insertions(+), 4 deletions(-) create mode 100644 examples/llm_plus_gnn/large_graph_indexer.py create mode 100644 examples/llm_plus_gnn/test_eval_gretriever.ipynb diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/examples/llm_plus_gnn/large_graph_indexer.py new file mode 100644 index 000000000000..c6bc2d3c592d --- /dev/null +++ b/examples/llm_plus_gnn/large_graph_indexer.py @@ -0,0 +1,109 @@ +from typing import Tuple, Hashable, Dict, Iterable, Set, Optional, Any +import pickle as pkl + +# Is there a multiprocessing-friendly implementation of this? + +TripletLike = Tuple[Hashable, Hashable, Hashable] + +class LargeGraphIndexer: + """ For a dataset that consists of mulitiple subgraphs that are assumed to + be part of a much larger graph, collate the values into a large graph store + to save resources + """ + + # TODO: Add support for Hetero graphs + + def __init__(self, nodes: Dict[Hashable, int], edges: Dict[TripletLike, int], node_attr: Optional[Dict[int, Dict[str, Any]]] = None, edge_attr: Optional[Dict[int, Dict[str, Any]]] = None) -> None: + self.nodes = nodes + self.edges = edges + + self.node_attr = node_attr or {v: dict(pid=k) for k, v in nodes.items()} + self.edge_attr = edge_attr or {v: dict(h=k[0], r=k[1], t=k[2]) for k, v in edges.items()} + + @classmethod + def from_triplets(cls, triplets: Iterable[TripletLike]) -> 'LargeGraphIndexer': + nodes = dict() + edges = dict() + + + for h, r, t in triplets: + nodes[h] = nodes.get(h) if h in nodes else len(nodes) + nodes[t] = nodes.get(t) if t in nodes else len(nodes) + edge_idx = (h, r, t) + edges[edge_idx] = edges.get(edge_idx) if edge_idx in edges else len(edges) + + return cls(nodes, edges) + + @classmethod + def collate(cls, graphs: Iterable['LargeGraphIndexer'], skip_shared_check=False) -> 'LargeGraphIndexer': + indexer = cls(dict(), dict()) + + for other_indexer in graphs: + if not skip_shared_check: + # figure out which ids to reindex + shared_nodes = other_indexer.nodes.keys() & indexer.nodes.keys() + + # This should be parallelizable + for node in shared_nodes: + indexer_node_attr = indexer.node_attr[indexer.nodes[node]] + other_indexer_node_attr = other_indexer.node_attr[other_indexer.nodes[node]].copy() + + # Assert that these are from the same graph + joint_keys = indexer_node_attr.keys() & other_indexer_node_attr.keys() + + for k, v in other_indexer_node_attr.items(): + if k in joint_keys and indexer_node_attr[k] != other_indexer_node_attr[k]: + raise AttributeError(f"Node features do not match! Only subgraphs from the same graph can be collated.") + indexer.node_attr[indexer.nodes[node]][k] = v + + new_nodes = other_indexer.nodes.keys() - indexer.nodes.keys() + + for node in new_nodes: + indexer.nodes[node] = len(indexer.nodes) + indexer.node_attr[indexer.nodes[node]] = other_indexer.node_attr[other_indexer.nodes[node]].copy() + + if not skip_shared_check: + + shared_edges = other_indexer.edges.keys() & indexer.edges.keys() + + # This should also be parallelizable + for edge in shared_edges: + indexer_edge_attr = indexer.edge_attr[indexer.edges[edge]] + other_indexer_edge_attr = other_indexer.edge_attr[other_indexer.edges[edge]] + + # Assert that these are from the same graph + joint_keys = indexer_edge_attr.keys() & other_indexer_edge_attr.keys() + + for k, v in other_indexer_edge_attr.items(): + if k in joint_keys and indexer_edge_attr[k] != other_indexer_edge_attr[k]: + raise AttributeError(f"Edge features do not match! Only subgraphs from the same graph can be collated.") + indexer.edge_attr[indexer.edges[edge]][k] = v + + new_edges = other_indexer.edges.keys() - indexer.edges.keys() + + for edge in new_edges: + indexer.edges[edge] = len(indexer.edges) + indexer.edge_attr[indexer.edges[edge]] = other_indexer.edge_attr[other_indexer.edges[edge]].copy() + + return indexer + + def get_unique_node_features(self, node_type: str) -> Set[Hashable]: + try: + return set([self.node_attr[i][node_type] for i in self.nodes.values()]) + except KeyError: + raise AttributeError(f"Nodes do not have a feature called {node_type}") + + def get_unique_edge_features(self, edge_type: str) -> Set[Hashable]: + try: + return set([self.edge_attr[i][edge_type] for i in self.edges.values()]) + except KeyError: + raise AttributeError(f"Edges do not have a feature called {edge_type}") + + def save(self, path: str) -> None: + with open(path, "w") as f: + pkl.dump(self, f) + + @classmethod + def from_disk(cls, path: str) -> 'LargeGraphIndexer': + with open(path) as f: + return pkl.load(f) \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_gretriever.ipynb new file mode 100644 index 000000000000..c6e1212e4c87 --- /dev/null +++ b/examples/llm_plus_gnn/test_eval_gretriever.ipynb @@ -0,0 +1,2011 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from abc import ABC, abstractmethod\n", + "from typing import Optional, Any, List, Iterable, Tuple, Dict, Union, Set\n", + "from collections.abc import Hashable\n", + "from torch_geometric.data import FeatureStore, InMemoryDataset, GraphStore\n", + "from torch_geometric.data.feature_store import TensorAttr, FeatureTensorType\n", + "from torch_geometric.distributed.local_graph_store import LocalGraphStore\n", + "from torch_geometric.datasets.web_qsp_dataset import WebQSPDataset\n", + "import torch\n", + "import datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class RawWebQSPDataset(WebQSPDataset):\n", + "\n", + " def __init__(\n", + " self,\n", + " root: str = \"\",\n", + " force_reload: bool = False,\n", + " ) -> None:\n", + " self._check_dependencies()\n", + " self.device = torch.device(\n", + " \"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " super(InMemoryDataset, self).__init__(root, None, None, force_reload=force_reload)\n", + " self._load_raw_data()\n", + " \n", + " @property\n", + " def raw_file_names(self) -> List[str]:\n", + " return [\"raw_data\", \"split_idxs\"]\n", + "\n", + " def _save_raw_data(self) -> None:\n", + " self.raw_dataset.save_to_disk(self.raw_paths[0])\n", + " torch.save(self.split_idxs, self.raw_paths[1]) \n", + "\n", + " def _load_raw_data(self) -> None:\n", + " self.raw_dataset = datasets.load_from_disk(self.raw_paths[0])\n", + " self.split_idxs = torch.load(self.raw_paths[1])\n", + " \n", + " def download(self) -> None:\n", + " super().download()\n", + " self._save_raw_data()\n", + "\n", + " def process(self) -> None:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing...\n", + "Done!\n" + ] + } + ], + "source": [ + "dataset = RawWebQSPDataset()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", + " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", + " ['Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", + " ['Record producer',\n", + " 'music.performance_role.regular_performances',\n", + " 'm.012m1vf1'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", + " ['2011 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0yrkr34'],\n", + " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Stratford', 'location.location.containedby', 'Canada'],\n", + " ['Singer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Musicians and Singers'],\n", + " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", + " ['As Long As You Love Me (Audien dubstep mix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kevin Risto', 'people.person.gender', 'Male'],\n", + " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", + " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", + " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", + " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", + " ['American Music Award for Favorite Pop/Rock Album',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc259'],\n", + " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", + " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", + " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Billboard Music Award for Top Streaming Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0njhx1b'],\n", + " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", + " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", + " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", + " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Results Show: Week 7'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['One Less Lonely Girl',\n", + " 'music.album.primary_release',\n", + " 'One Less Lonely Girl'],\n", + " ['Twista', 'people.person.gender', 'Male'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", + " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", + " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", + " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", + " ['Madonna', 'music.artist.genre', 'Pop music'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", + " ['m.0gbm3cg',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", + " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", + " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", + " ['MTV Video Music Award Japan for Best New Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhrwc'],\n", + " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", + " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", + " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['m.0gctwjk', 'tv.tv_guest_role.episodes_appeared_in', 'Series 2, Episode 3'],\n", + " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", + " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", + " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", + " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", + " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Beauty and a Beat',\n", + " 'music.single.versions',\n", + " 'Beauty and a Beat (Wideboys Club Mix)'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Bryan Adams',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male'],\n", + " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", + " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", + " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'music.composition.form', 'Song'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0yrkr34'],\n", + " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", + " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", + " ['Record producer',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Haley James Scott'],\n", + " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", + " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['Kanye West', 'people.person.profession', 'Singer'],\n", + " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", + " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", + " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", + " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", + " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", + " ['RedOne', 'music.artist.genre', 'Rock music'],\n", + " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", + " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", + " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'music.recording.featured_artists',\n", + " 'Big Sean'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", + " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", + " ['Athan Grace', 'people.person.profession', 'Actor'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", + " ['All Around the World (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", + " ['Brandy Norwood',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Whitney Houston'],\n", + " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['MTV Video Music Award for Artist to Watch',\n", + " 'award.award_category.winners',\n", + " 'm.0n1ykxp'],\n", + " ['Caitlin Beadles',\n", + " 'celebrities.celebrity.sexual_relationships',\n", + " 'm.0d33gyj'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audiobot instrumental)'],\n", + " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", + " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'The Mighty Mighty Bosstones'],\n", + " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", + " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", + " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Documentary film'],\n", + " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", + " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Amerie', 'music.artist.genre', 'Pop music'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", + " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep edit)'],\n", + " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Recovery', 'music.recording.song', 'Recovery'],\n", + " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", + " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", + " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", + " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", + " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", + " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", + " ['#thatPower', 'music.recording.song', '#thatPower'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", + " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['2013 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0wjgqck'],\n", + " ['The Island Def Jam Music Group',\n", + " 'organization.organization.parent',\n", + " 'm.04q65lb'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Rusted Root'],\n", + " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['m.0z87d3n',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", + " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", + " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", + " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", + " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", + " ['Tampa', 'location.location.containedby', 'United States of America'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.compositions',\n", + " 'Love Never Felt So Good'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", + " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", + " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", + " ['Estelle', 'people.person.profession', 'Record producer'],\n", + " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", + " ['Chris Jasper', 'people.person.gender', 'Male'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", + " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", + " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", + " ['Snoop Dogg',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['David Nicksay', 'people.person.gender', 'Male'],\n", + " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", + " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Juno Awards of 2014',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0102z0vx'],\n", + " ['As Long As You Love Me (Audiobot remix)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", + " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", + " ['m.0_cyzs_',\n", + " 'celebrities.legal_entanglement.offense',\n", + " 'Driving under the influence'],\n", + " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", + " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", + " ['Mann', 'people.person.gender', 'Male'],\n", + " ['JoJo', 'people.person.gender', 'Female'],\n", + " ['Right Here (featuring Drake)',\n", + " 'music.recording.canonical_version',\n", + " 'Right Here'],\n", + " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", + " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0yrjynf',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", + " ['Pras', 'people.person.profession', 'Record producer'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", + " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", + " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", + " ['Marvin Isley',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['School Gyrls', 'film.film.language', 'English Language'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", + " ['Katy Perry', 'people.person.profession', 'Actor'],\n", + " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", + " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.featured_artists',\n", + " 'Redfoo'],\n", + " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", + " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", + " ['As Long as You Love Me (album version)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Justin Bieber: Just Getting Started',\n", + " 'book.written_work.author',\n", + " 'Justin Bieber'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Marc Maris vs. Ramone'],\n", + " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", + " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", + " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", + " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.releases',\n", + " 'Love Never Felt So Good'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", + " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", + " ['Dance music',\n", + " 'broadcast.genre.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", + " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", + " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", + " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", + " ['Scooter Braun', 'people.person.gender', 'Male'],\n", + " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", + " ['Sir Nolan', 'people.person.gender', 'Male'],\n", + " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", + " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", + " ['Adam Messinger',\n", + " 'music.composer.compositions',\n", + " \"Turn to You (Mother's Day Dedication)\"],\n", + " ['m.0yrktlv',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Male Hottie'],\n", + " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", + " ['Iggy Azalea',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", + " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['m.0gxnnzy', 'celebrities.romantic_relationship.relationship_type', 'Dated'],\n", + " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", + " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", + " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Britney Spears',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", + " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", + " ['Fergie', 'music.artist.genre', 'Rock music'],\n", + " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", + " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", + " ['Mistletoe', 'music.album.release_type', 'Single'],\n", + " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", + " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.tracks',\n", + " 'Live My Life (Party Rock remix)'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['m.0njw4z2',\n", + " 'award.award_honor.award',\n", + " 'MTV Europe Music Award for Best Male'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", + " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", + " ['m.0gbm3c3',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Miley Cyrus'],\n", + " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", + " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", + " ['JoJo', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Next to You', 'music.album.release_type', 'Single'],\n", + " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", + " ['Willa Ford', 'people.person.languages', 'English Language'],\n", + " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", + " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", + " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", + " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Hot Wired Radio',\n", + " 'broadcast.content.broadcast',\n", + " 'Hot Wired Radio - 128kbps Stream'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", + " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0gbm3d9',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", + " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", + " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", + " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", + " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", + " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", + " ['Justin Timberlake',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Khalil', 'people.person.gender', 'Male'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", + " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", + " ['Selena Gomez',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", + " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", + " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['C1', 'music.artist.genre', 'Hip hop music'],\n", + " ['My Worlds Acoustic',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Album content type'],\n", + " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", + " ['Live My Life', 'music.composition.language', 'English Language'],\n", + " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", + " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", + " ['Baby', 'music.album.releases', 'Baby'],\n", + " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", + " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", + " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", + " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", + " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", + " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", + " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.production_companies',\n", + " 'AEG Live'],\n", + " ['Redfoo', 'people.person.gender', 'Male'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", + " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", + " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", + " ['m.01053qzf',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", + " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", + " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", + " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", + " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", + " ['Ray J', 'people.person.profession', 'Musician'],\n", + " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", + " ['m.0101fv5f',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", + " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", + " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", + " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", + " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Guest host'],\n", + " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", + " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", + " ['Christina Milian', 'people.person.profession', 'Actor'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", + " ['Dance music', 'broadcast.genre.content', '181-party'],\n", + " ['Queen Elizabeth II Diamond Jubilee Medal',\n", + " 'award.award_category.winners',\n", + " 'm.0njwqrb'],\n", + " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", + " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", + " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['CMT Music Award: Collaborative Video of the Year',\n", + " 'award.award_category.winners',\n", + " 'm.0njvs9s'],\n", + " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", + " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", + " ['Chingy', 'people.person.profession', 'Actor'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", + " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", + " ['Justin Bieber',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", + " ['Chingy', 'people.person.gender', 'Male'],\n", + " ['Reed Smoot', 'people.person.gender', 'Male'],\n", + " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", + " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.releases',\n", + " 'As Long As You Love Me (remixes)'],\n", + " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjvlh'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", + " ['Singer', 'common.topic.article', 'm.09l6h'],\n", + " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", + " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0pc670l'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", + " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", + " ['Beyoncé Knowles',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['m.0njhyh_',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Streaming Song (Video)'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", + " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_production_design_by',\n", + " 'Devorah Herbert'],\n", + " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", + " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", + " ['Nick Jonas',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", + " ['Lupe Fiasco',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['Martin Kierszenbaum',\n", + " 'people.person.place_of_birth',\n", + " 'United States of America'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long as You Love Me'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", + " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", + " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", + " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", + " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", + " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0y5tl39',\n", + " 'film.personal_film_appearance.film',\n", + " 'Les Coulisses des Golden Globes'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", + " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", + " ['justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z0tgz6'],\n", + " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", + " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", + " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", + " ['Everlast', 'music.artist.label', 'Island Records'],\n", + " ['5th Annual Shorty Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0ywvh8k'],\n", + " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Baby', 'common.topic.notable_types', 'Composition'],\n", + " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['m.0tkqqgg',\n", + " 'award.award_nomination.award',\n", + " 'Juno Award for Pop Album of the Year'],\n", + " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Nasri', 'people.person.profession', 'Singer'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.compositions',\n", + " 'As Long as You Love Me'],\n", + " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", + " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", + " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", + " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", + " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", + " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", + " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", + " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", + " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", + " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", + " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.film',\n", + " 'Zendaya: Behind the Scenes'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", + " ['That Should Be Me', 'music.composition.form', 'Song'],\n", + " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", + " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Justin Bieber'],\n", + " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.song',\n", + " 'Beauty And A Beat'],\n", + " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", + " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Usher', 'music.composer.compositions', 'First Dance'],\n", + " ['m.0n1ykxp',\n", + " 'award.award_honor.award',\n", + " 'MTV Video Music Award for Artist to Watch'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Rockumentary'],\n", + " ['Amerie', 'people.person.gender', 'Female'],\n", + " ['Real Change: Artists for Education',\n", + " 'film.film.personal_appearances',\n", + " 'm.0y5th3r'],\n", + " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", + " ['Beautiful and the Beat',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['PYD', 'common.topic.notable_types', 'Composition'],\n", + " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", + " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", + " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", + " ['iJustine', 'people.person.gender', 'Female'],\n", + " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", + " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", + " ['m.0ng_vkd',\n", + " 'film.personal_film_appearance.film',\n", + " 'A Michael Bublé Christmas'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", + " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", + " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", + " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Vanessa Hudgens',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", + " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", + " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Madonna', 'people.person.profession', 'Record producer'],\n", + " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", + " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", + " ['As Long as You Love Me (acoustic version)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", + " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", + " ['Rob Thomas', 'people.person.gender', 'Male'],\n", + " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", + " ['m.0hvlt03',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", + " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_srv2b'],\n", + " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", + " ['Donna Summer',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0101fszs',\n", + " 'film.personal_film_appearance.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Alanis Morissette',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Official website'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Jenna Andrews'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", + " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Ray J', 'people.person.nationality', 'United States of America'],\n", + " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Nelly Furtado',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['m.0gbm3fj',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", + " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", + " ['As Long As You Love Me (Ferry Corsten club dub)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", + " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", + " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", + " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", + " ['Guest host',\n", + " 'tv.non_character_role.tv_guest_personal_appearances',\n", + " 'm.0v_98y7'],\n", + " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", + " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", + " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", + " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", + " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", + " ['L.A. Reid', 'people.person.gender', 'Male'],\n", + " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['London', 'location.location.containedby', 'Ontario'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_set_decoration_by',\n", + " 'Lia Roldan'],\n", + " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", + " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Change Me', 'music.album.releases', 'Change Me'],\n", + " ['RedOne', 'music.artist.label', 'Island Records'],\n", + " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", + " ['All Around the World',\n", + " 'music.recording.canonical_version',\n", + " 'All Around the World'],\n", + " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0wjhc6c'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", + " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", + " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Lupe Fiasco'],\n", + " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", + " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", + " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", + " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", + " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Beauty and a Beat (Wideboys Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", + " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", + " ['m.0v90skf',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Male Artist'],\n", + " ['Ludacris', 'people.person.profession', 'Actor'],\n", + " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", + " ['Cameo appearance',\n", + " 'tv.special_tv_performance_type.episode_performances',\n", + " 'm.0v1lwt2'],\n", + " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", + " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'One Chance'],\n", + " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", + " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", + " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", + " ['Roller Coaster',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_x4zg3'],\n", + " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", + " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", + " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", + " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", + " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", + " ['m.0sgk_cw',\n", + " 'award.award_honor.award',\n", + " \"Kids' Choice Award for Favorite Song\"],\n", + " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", + " ['MTV Video Music Brazil Award for Best International Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhhqv'],\n", + " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", + " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", + " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", + " ['Britney Spears', 'people.person.profession', 'Singer'],\n", + " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", + " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Rock music',\n", + " 'base.webvideo.internet_video_genre.series',\n", + " 'Biscuithands, The Animated Musical'],\n", + " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", + " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Barry Weiss'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['MTV Europe Music Award for Best Male',\n", + " 'award.award_category.winners',\n", + " 'm.0z1scxk'],\n", + " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", + " ['Will Smith', 'people.person.profession', 'Actor'],\n", + " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", + " ['Will i Am', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Fabian', 'people.person.gender', 'Male'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", + " ['Somebody to Love (remix)',\n", + " 'music.album.primary_release',\n", + " 'Somebody to Love (remix)'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", + " ['Spouse', 'type.property.expected_type', 'Person'],\n", + " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", + " ['Baby', 'music.recording.artist', 'Ludacris'],\n", + " ['Rudolph Valentino',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", + " ['Judy Garland',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Kelly Clarkson',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'm.011h9_22'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", + " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", + " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0vp8rhw',\n", + " 'music.recording_contribution.album',\n", + " 'Beauty and a Beat (Remixes)'],\n", + " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", + " ['Katy Perry: Part of Me',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['m.0jsmvv5',\n", + " 'film.film_regional_release_date.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", + " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", + " ['All Around The World (featuring Ludacris)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", + " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", + " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", + " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", + " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", + " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep mix)'],\n", + " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Fergie', 'people.person.profession', 'Actor'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", + " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", + " ['Zac Efron', 'people.person.gender', 'Male'],\n", + " ['P!nk', 'music.artist.genre', 'Rock music'],\n", + " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", + " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", + " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", + " ['Under the Mistletoe',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Slick Rick'],\n", + " ['Amerie', 'music.artist.genre', 'Rock music'],\n", + " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0pbzq13',\n", + " 'film.performance.special_performance_type',\n", + " 'Cameo appearance'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", + " ['Height', 'type.property.unit', 'Meter'],\n", + " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", + " ['Ray J',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Boyfriend (acoustic version)',\n", + " 'music.recording.canonical_version',\n", + " 'Boyfriend'],\n", + " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Believe Tour', 'music.concert_tour.album_or_release_supporting', 'Believe'],\n", + " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", + " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", + " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", + " ['Frank Ocean',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['As Long As You Love Me (Audiobot instrumental)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Usher', 'people.person.nationality', 'United States of America'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", + " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", + " ['m.0gbm3b7',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Sheryl Crow',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['All That Matters',\n", + " 'music.composition.composer',\n", + " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Nasri', 'music.artist.genre', 'Reggae'],\n", + " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", + " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", + " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", + " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", + " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", + " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", + " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long As You Love Me'],\n", + " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", + " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", + " ['Whitney Houston', 'people.person.profession', 'Model'],\n", + " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['As Long as You Love Me',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Disney Parks Christmas Day Parade',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['Ray J', 'people.person.profession', 'Artist'],\n", + " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", + " ['American Music Award for Favorite Pop/Rock Male Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc0sf'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", + " ['The Notorious B.I.G.',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", + " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", + " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", + " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", + " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Haley James Scott',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Record producer'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", + " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", + " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['My Worlds: The Collection',\n", + " 'music.album.album_content_type',\n", + " 'Compilation album'],\n", + " ['Lolly', 'music.album.releases', 'Lolly'],\n", + " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", + " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", + " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", + " ['Ja Rule', 'people.person.profession', 'Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", + " ['Die in Your Arms',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z85qxq'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", + " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", + " ['Kuk Harrell',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0101ft5f'],\n", + " ['Somebody to Love (J Stax remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'film.film.executive_produced_by',\n", + " 'Allison Kaye Scarinzi'],\n", + " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", + " ['Nasri', 'music.artist.genre', 'Pop music'],\n", + " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", + " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", + " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.05sp405',\n", + " 'organization.organization_relationship.child',\n", + " 'Island Records'],\n", + " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", + " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", + " ['John Mamann',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Teen Choice Award for Choice Summer Music Star: Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjynf'],\n", + " ['Juicy J', 'people.person.profession', 'Actor'],\n", + " ['m.0yqflrk',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'celebritynetworth.com'],\n", + " ['Miley Cyrus',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", + " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['m.04q65lb',\n", + " 'organization.organization_relationship.child',\n", + " 'The Island Def Jam Music Group'],\n", + " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", + " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", + " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['My World', 'music.album.artist', 'Justin Bieber'],\n", + " ['Don Henley', 'people.person.gender', 'Male'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Musician', 'people.profession.specializations', 'Singer'],\n", + " ['Die in Your Arms', 'music.recording.canonical_version', 'Die in Your Arms'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['m.0njvs9s',\n", + " 'award.award_honor.award',\n", + " 'CMT Music Award: Collaborative Video of the Year'],\n", + " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber',\n", + " 'music.artist.album',\n", + " 'Turn to You (Mother’s Day Dedication)'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", + " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['City/Town/Village', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", + " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Terence Dudley', 'people.person.gender', 'Male'],\n", + " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Mr. Sosa & The Yayo'],\n", + " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", + " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", + " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0101fvbf',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['m.0yrhrwc', 'award.award_honor.ceremony', '2011 MTV Video Music Aid Japan'],\n", + " ['MTV Europe Music Award for Best North American Act',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhmll'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['m.0101ft1d',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", + " ['Ciara', 'people.person.gender', 'Female'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", + " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", + " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0jzrrqs', 'location.mailing_address.country', 'United States of America'],\n", + " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", + " ['Big R Radio - Top 40 Hits',\n", + " 'broadcast.content.artist',\n", + " 'Natasha Bedingfield'],\n", + " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ...]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.raw_dataset[0]['graph']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "test_graph_store = LocalGraphStore()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attr = dict(\n", + " edge_type=None,\n", + " layout='coo',\n", + " size=(2,2),\n", + " is_sorted=False\n", + ")\n", + "test_graph_store.put_edge_index(torch.Tensor([[0,1], [1,2]]), **attr)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[EdgeAttr(edge_type=None, layout=, is_sorted=False, size=(2, 2))]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_graph_store.get_all_edge_attrs()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from large_graph_indexer import LargeGraphIndexer" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "indexer = LargeGraphIndexer.from_triplets(dataset.raw_dataset[0]['graph'])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "indexer2 = LargeGraphIndexer.from_triplets(dataset.raw_dataset[1]['graph'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "bigger_indexer = LargeGraphIndexer.collate([indexer, indexer2])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(indexer.nodes) + len(indexer2.nodes) - len(indexer.nodes.keys() & indexer2.nodes.keys()) == len(bigger_indexer.nodes)\n", + "assert len(indexer.edges) + len(indexer2.edges) - len(indexer.edges.keys() & indexer2.edges.keys()) == len(bigger_indexer.edges)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(set(bigger_indexer.nodes.values())) == len(bigger_indexer.nodes)\n", + "assert len(set(bigger_indexer.edges.values())) == len(bigger_indexer.edges)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "for node in indexer.nodes.keys():\n", + " assert indexer.node_attr[indexer.nodes[node]][\"pid\"] == node, f'{node} is not {indexer.node_attr[indexer.nodes[node]][\"pid\"]}'" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "import tqdm\n", + "from multiprocessing import Pool" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#TODO: can probably be parallelized\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Pool(\u001b[38;5;241m40\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m p:\n\u001b[0;32m----> 4\u001b[0m indexers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(tqdm\u001b[38;5;241m.\u001b[39mtqdm(p\u001b[38;5;241m.\u001b[39mimap(LargeGraphIndexer\u001b[38;5;241m.\u001b[39mfrom_triplets, [ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgraph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m dataset\u001b[38;5;241m.\u001b[39mraw_dataset]), total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset\u001b[38;5;241m.\u001b[39mraw_dataset)))\n", + "Cell \u001b[0;32mIn[16], line 4\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#TODO: can probably be parallelized\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Pool(\u001b[38;5;241m40\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m p:\n\u001b[0;32m----> 4\u001b[0m indexers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(tqdm\u001b[38;5;241m.\u001b[39mtqdm(p\u001b[38;5;241m.\u001b[39mimap(LargeGraphIndexer\u001b[38;5;241m.\u001b[39mfrom_triplets, [ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgraph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m dataset\u001b[38;5;241m.\u001b[39mraw_dataset]), total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset\u001b[38;5;241m.\u001b[39mraw_dataset)))\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/arrow_dataset.py:2440\u001b[0m, in \u001b[0;36mDataset.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 2438\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(pa_subtable\u001b[38;5;241m.\u001b[39mnum_rows):\n\u001b[1;32m 2439\u001b[0m pa_subtable_ex \u001b[38;5;241m=\u001b[39m pa_subtable\u001b[38;5;241m.\u001b[39mslice(i, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m-> 2440\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformat_table\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2441\u001b[0m \u001b[43m \u001b[49m\u001b[43mpa_subtable_ex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2442\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2443\u001b[0m \u001b[43m \u001b[49m\u001b[43mformatter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformatter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2444\u001b[0m \u001b[43m \u001b[49m\u001b[43mformat_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_format_columns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2445\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_all_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_output_all_columns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2446\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2447\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m formatted_output\n\u001b[1;32m 2448\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:641\u001b[0m, in \u001b[0;36mformat_table\u001b[0;34m(table, key, formatter, format_columns, output_all_columns)\u001b[0m\n\u001b[1;32m 639\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 640\u001b[0m pa_table_to_format \u001b[38;5;241m=\u001b[39m pa_table\u001b[38;5;241m.\u001b[39mdrop(col \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m pa_table\u001b[38;5;241m.\u001b[39mcolumn_names \u001b[38;5;28;01mif\u001b[39;00m col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m format_columns)\n\u001b[0;32m--> 641\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformatter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table_to_format\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_type\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 642\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_all_columns:\n\u001b[1;32m 643\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(formatted_output, MutableMapping):\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:397\u001b[0m, in \u001b[0;36mFormatter.__call__\u001b[0;34m(self, pa_table, query_type)\u001b[0m\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable, query_type: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[RowFormat, ColumnFormat, BatchFormat]:\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrow\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 397\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat_column(pa_table)\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:437\u001b[0m, in \u001b[0;36mPythonFormatter.format_row\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlazy:\n\u001b[1;32m 436\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m LazyRow(pa_table, \u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m--> 437\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpython_arrow_extractor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 438\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpython_features_decoder\u001b[38;5;241m.\u001b[39mdecode_row(row)\n\u001b[1;32m 439\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m row\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:145\u001b[0m, in \u001b[0;36mPythonArrowExtractor.extract_row\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mextract_row\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[0;32m--> 145\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _unnest(\u001b[43mpa_table\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_pydict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "indexers = []\n", + "#TODO: can probably be parallelized\n", + "with Pool(40) as p:\n", + " indexers = list(tqdm.tqdm(p.imap(LargeGraphIndexer.from_triplets, [ds['graph'] for ds in dataset.raw_dataset]), total=len(dataset.raw_dataset)))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'indexers' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#FIXME: right now this is really slow\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m big_indexer \u001b[38;5;241m=\u001b[39m LargeGraphIndexer\u001b[38;5;241m.\u001b[39mcollate(tqdm\u001b[38;5;241m.\u001b[39mtqdm(\u001b[43mindexers\u001b[49m), skip_shared_check\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "\u001b[0;31mNameError\u001b[0m: name 'indexers' is not defined" + ] + } + ], + "source": [ + "#FIXME: right now this is really slow\n", + "big_indexer = LargeGraphIndexer.collate(tqdm.tqdm(indexers), skip_shared_check=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "from itertools import chain" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "large_graph_dataset = chain.from_iterable([ds['graph'] for ds in dataset.raw_dataset])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 4700/4700 [00:51<00:00, 92.08it/s] \n" + ] + } + ], + "source": [ + "total_size = 0\n", + "for ds in tqdm.tqdm(dataset.raw_dataset):\n", + " total_size += len(ds['graph'])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 19986134/19986134 [00:20<00:00, 956393.67it/s] \n" + ] + } + ], + "source": [ + "large_indexer = LargeGraphIndexer.from_triplets(tqdm.tqdm(large_graph_dataset, total=total_size))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "node_attributes = list(large_indexer.get_unique_node_features(\"pid\"))\n", + "node_attributes = [i.lower() for i in node_attributes]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "edge_attributes = list(large_indexer.get_unique_edge_features(\"r\"))\n", + "edge_attributes = [i.lower() for i in edge_attributes]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6094" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(edge_attributes)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.nn.text import text2embedding, SentenceTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inherit model weights from sentence-transformers/all-roberta-large-v1\n" + ] + }, + { + "data": { + "text/plain": [ + "SentenceTransformer(\n", + " (bert_model): RobertaModel(\n", + " (embeddings): RobertaEmbeddings(\n", + " (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n", + " (position_embeddings): Embedding(514, 1024, padding_idx=1)\n", + " (token_type_embeddings): Embedding(1, 1024)\n", + " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): RobertaEncoder(\n", + " (layer): ModuleList(\n", + " (0-23): 24 x RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (key): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (value): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (intermediate_act_fn): GELUActivation()\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): RobertaPooler(\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(\"sentence-transformers/all-roberta-large-v1\").to(device)\n", + "model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuda\n" + ] + } + ], + "source": [ + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "from more_itertools import chunked" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/5071 [00:00 None: + self._check_dependencies() + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + super().__init__(root, None, None, force_reload=force_reload) + self.load(self.processed_paths[0]) + + def _check_dependencies(self) -> None: + missing_imports = False missing_str_list = [] if not WITH_PCST: missing_str_list.append('pcst_fast') @@ -172,10 +180,6 @@ def __init__( missing_str = ' '.join(missing_str_list) error_out = f"`pip install {missing_str}` to use this dataset." raise ImportError(error_out) - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") - super().__init__(root, None, None, force_reload=force_reload) - self.load(self.processed_paths[0]) @property def raw_file_names(self) -> List[str]: From f109bf9295ba9dfb55af16f3f5ae86fe09e6013a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 28 May 2024 19:39:09 -0700 Subject: [PATCH 583/752] dataloading and retrieval debugging --- examples/llm_plus_gnn/large_graph_indexer.py | 314 ++- examples/llm_plus_gnn/raw_qsp_dataset.py | 103 + .../llm_plus_gnn/test_eval_gretriever.ipynb | 2093 ++++------------- 3 files changed, 827 insertions(+), 1683 deletions(-) create mode 100644 examples/llm_plus_gnn/raw_qsp_dataset.py diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/examples/llm_plus_gnn/large_graph_indexer.py index c6bc2d3c592d..4f8c700d4311 100644 --- a/examples/llm_plus_gnn/large_graph_indexer.py +++ b/examples/llm_plus_gnn/large_graph_indexer.py @@ -1,109 +1,257 @@ -from typing import Tuple, Hashable, Dict, Iterable, Set, Optional, Any +from typing import ( + Tuple, + Hashable, + Dict, + Iterable, + Set, + Optional, + Any, + Iterator, + List, + Sequence, +) import pickle as pkl +from dataclasses import dataclass +from itertools import chain # Is there a multiprocessing-friendly implementation of this? TripletLike = Tuple[Hashable, Hashable, Hashable] + +def ordered_set(values: Iterable[Hashable]) -> List[Hashable]: + return list(dict.fromkeys(values)) + + +NODE_PID = "pid" + +EDGE_PID = "e_pid" +EDGE_HEAD = "h" +EDGE_RELATION = "r" +EDGE_TAIL = "t" + + +@dataclass +class MappedFeature: + name: str + values: Sequence[Any] + + class LargeGraphIndexer: - """ For a dataset that consists of mulitiple subgraphs that are assumed to - be part of a much larger graph, collate the values into a large graph store - to save resources + """For a dataset that consists of mulitiple subgraphs that are assumed to + be part of a much larger graph, collate the values into a large graph store + to save resources """ - # TODO: Add support for Hetero graphs + def __init__( + self, + nodes: Iterable[Hashable], + edges: Iterable[TripletLike], + node_attr: Optional[Dict[str, List[Any]]] = None, + edge_attr: Optional[Dict[str, List[Any]]] = None, + ) -> None: + self._nodes: Dict[Hashable, int] = dict() + self._edges: Dict[TripletLike, int] = dict() - def __init__(self, nodes: Dict[Hashable, int], edges: Dict[TripletLike, int], node_attr: Optional[Dict[int, Dict[str, Any]]] = None, edge_attr: Optional[Dict[int, Dict[str, Any]]] = None) -> None: - self.nodes = nodes - self.edges = edges + self._mapped_node_features: Set[str] = set() + self._mapped_edge_features: Set[str] = set() - self.node_attr = node_attr or {v: dict(pid=k) for k, v in nodes.items()} - self.edge_attr = edge_attr or {v: dict(h=k[0], r=k[1], t=k[2]) for k, v in edges.items()} + if len(nodes) != len(set(nodes)): + raise AttributeError("Nodes need to be unique") + if len(edges) != len(set(edges)): + raise AttributeError("Edges need to be unique") - @classmethod - def from_triplets(cls, triplets: Iterable[TripletLike]) -> 'LargeGraphIndexer': - nodes = dict() - edges = dict() + if node_attr is not None: + # TODO: Validity checks btw nodes and node_attr + self.node_attr = node_attr + else: + self.node_attr = dict() + self.node_attr[NODE_PID] = nodes + + for i, node in enumerate(self.node_attr[NODE_PID]): + self._nodes[node] = i + if edge_attr is not None: + # TODO: Validity checks btw edges and edge_attr + self.edge_attr = edge_attr + for i, tup in enumerate(edges): + self._edges[tup] = i + else: + self.edge_attr = dict() + for default_key in [EDGE_HEAD, EDGE_RELATION, EDGE_TAIL, EDGE_PID]: + self.edge_attr[default_key] = list() + + for i, tup in enumerate(edges): + h, r, t = tup + self.edge_attr[EDGE_HEAD].append(h) + self.edge_attr[EDGE_RELATION].append(r) + self.edge_attr[EDGE_TAIL].append(t) + self.edge_attr[EDGE_PID].append(tup) + self._edges[tup] = i + + @classmethod + def from_triplets(cls, triplets: Iterable[TripletLike]) -> "LargeGraphIndexer": + # NOTE: Right now assumes that all trips can be loaded into memory + nodes = set() + edges = set() for h, r, t in triplets: - nodes[h] = nodes.get(h) if h in nodes else len(nodes) - nodes[t] = nodes.get(t) if t in nodes else len(nodes) + + for node in (h, t): + nodes.add(node) + edge_idx = (h, r, t) - edges[edge_idx] = edges.get(edge_idx) if edge_idx in edges else len(edges) - - return cls(nodes, edges) - + edges.add(edge_idx) + + return cls(list(nodes), list(edges)) + @classmethod - def collate(cls, graphs: Iterable['LargeGraphIndexer'], skip_shared_check=False) -> 'LargeGraphIndexer': - indexer = cls(dict(), dict()) - - for other_indexer in graphs: - if not skip_shared_check: - # figure out which ids to reindex - shared_nodes = other_indexer.nodes.keys() & indexer.nodes.keys() - - # This should be parallelizable - for node in shared_nodes: - indexer_node_attr = indexer.node_attr[indexer.nodes[node]] - other_indexer_node_attr = other_indexer.node_attr[other_indexer.nodes[node]].copy() - - # Assert that these are from the same graph - joint_keys = indexer_node_attr.keys() & other_indexer_node_attr.keys() - - for k, v in other_indexer_node_attr.items(): - if k in joint_keys and indexer_node_attr[k] != other_indexer_node_attr[k]: - raise AttributeError(f"Node features do not match! Only subgraphs from the same graph can be collated.") - indexer.node_attr[indexer.nodes[node]][k] = v - - new_nodes = other_indexer.nodes.keys() - indexer.nodes.keys() - - for node in new_nodes: - indexer.nodes[node] = len(indexer.nodes) - indexer.node_attr[indexer.nodes[node]] = other_indexer.node_attr[other_indexer.nodes[node]].copy() - - if not skip_shared_check: - - shared_edges = other_indexer.edges.keys() & indexer.edges.keys() - - # This should also be parallelizable - for edge in shared_edges: - indexer_edge_attr = indexer.edge_attr[indexer.edges[edge]] - other_indexer_edge_attr = other_indexer.edge_attr[other_indexer.edges[edge]] - - # Assert that these are from the same graph - joint_keys = indexer_edge_attr.keys() & other_indexer_edge_attr.keys() - - for k, v in other_indexer_edge_attr.items(): - if k in joint_keys and indexer_edge_attr[k] != other_indexer_edge_attr[k]: - raise AttributeError(f"Edge features do not match! Only subgraphs from the same graph can be collated.") - indexer.edge_attr[indexer.edges[edge]][k] = v - - new_edges = other_indexer.edges.keys() - indexer.edges.keys() - - for edge in new_edges: - indexer.edges[edge] = len(indexer.edges) - indexer.edge_attr[indexer.edges[edge]] = other_indexer.edge_attr[other_indexer.edges[edge]].copy() - - return indexer - - def get_unique_node_features(self, node_type: str) -> Set[Hashable]: + def collate(cls, graphs: Iterable["LargeGraphIndexer"]) -> "LargeGraphIndexer": + # FIXME Needs to merge node attrs and edge attrs? + trips = chain.from_iterable([graph.to_triplets() for graph in graphs]) + return cls.from_triplets(trips) + + def get_unique_node_features(self, feature_name: str = NODE_PID) -> List[Hashable]: try: - return set([self.node_attr[i][node_type] for i in self.nodes.values()]) + if feature_name in self._mapped_node_features: + raise IndexError(f"Only non-mapped features can be retrieved uniquely.") + return ordered_set(self.get_node_features(feature_name)) + except KeyError: - raise AttributeError(f"Nodes do not have a feature called {node_type}") - - def get_unique_edge_features(self, edge_type: str) -> Set[Hashable]: + raise AttributeError(f"Nodes do not have a feature called {feature_name}") + + def add_node_feature( + self, + new_feature_name: str, + new_feature_vals: Sequence[Any], + map_from_feature: str = NODE_PID, + ) -> None: + + if new_feature_name in self.node_attr: + raise AttributeError("Features cannot be overridden once created") + if map_from_feature in self._mapped_node_features: + raise AttributeError(f"{map_from_feature} is already a feature mapping.") + + feature_keys = self.get_unique_node_features(map_from_feature) + if len(feature_keys) != len(new_feature_vals): + raise AttributeError( + f"Expected encodings for {len(feature_keys)} unique features, but got {len(new_feature_vals)} encodings." + ) + + if map_from_feature == NODE_PID: + self.node_attr[new_feature_name] = new_feature_vals + else: + self.node_attr[new_feature_name] = MappedFeature( + name=map_from_feature, values=new_feature_vals + ) + self._mapped_node_features.add(new_feature_name) + + def get_node_features( + self, feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None + ) -> List[Any]: + return list(self.get_node_features_iter(feature_name, pids)) + + def get_node_features_iter( + self, feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None + ) -> Iterator[Any]: + if pids is None: + pids = self.node_attr[NODE_PID] + + if feature_name in self._mapped_node_features: + feature_map_info = self.node_attr[feature_name] + from_feature_name, to_feature_vals = ( + feature_map_info.name, + feature_map_info.values, + ) + from_feature_vals = self.get_unique_node_features(from_feature_name) + feature_mapping = {k: i for i, k in enumerate(from_feature_vals)} + + for pid in pids: + idx = self._nodes[pid] + from_feature_val = self.node_attr[from_feature_name][idx] + to_feature_idx = feature_mapping[from_feature_val] + yield to_feature_vals[to_feature_idx] + else: + for pid in pids: + idx = self._nodes[pid] + yield self.node_attr[feature_name][idx] + + def get_unique_edge_features(self, feature_name: str = EDGE_PID) -> List[Hashable]: try: - return set([self.edge_attr[i][edge_type] for i in self.edges.values()]) + if feature_name in self._mapped_edge_features: + raise IndexError(f"Only non-mapped features can be retrieved uniquely.") + return ordered_set(self.get_edge_features(feature_name)) except KeyError: - raise AttributeError(f"Edges do not have a feature called {edge_type}") - + raise AttributeError(f"Edges do not have a feature called {feature_name}") + + def add_edge_feature( + self, + new_feature_name: str, + new_feature_vals: Sequence[Any], + map_from_feature: str = EDGE_PID, + ) -> None: + + if new_feature_name in self.edge_attr: + raise AttributeError("Features cannot be overridden once created") + if map_from_feature in self._mapped_edge_features: + raise AttributeError(f"{map_from_feature} is already a feature mapping.") + + feature_keys = self.get_unique_edge_features(map_from_feature) + if len(feature_keys) != len(new_feature_vals): + raise AttributeError( + f"Expected encodings for {len(feature_keys)} unique features, but got {len(new_feature_vals)} encodings." + ) + + if map_from_feature == EDGE_PID: + self.edge_attr[new_feature_name] = new_feature_vals + else: + self.edge_attr[new_feature_name] = MappedFeature( + name=map_from_feature, values=new_feature_vals + ) + self._mapped_edge_features.add(new_feature_name) + + def get_edge_features( + self, + feature_name: str = EDGE_RELATION, + pids: Optional[Iterable[Hashable]] = None, + ) -> List[Any]: + return list(self.get_edge_features_iter(feature_name, pids)) + + def get_edge_features_iter( + self, + feature_name: str = EDGE_RELATION, + pids: Optional[Iterable[TripletLike]] = None, + ) -> Iterator[Any]: + if pids is None: + pids = self.edge_attr[EDGE_PID] + + if feature_name in self._mapped_edge_features: + feature_map_info = self.edge_attr[feature_name] + from_feature_name, to_feature_vals = ( + feature_map_info.name, + feature_map_info.values, + ) + from_feature_vals = self.get_unique_edge_features(from_feature_name) + feature_mapping = {k: i for i, k in enumerate(from_feature_vals)} + + for pid in pids: + idx = self._edges[pid] + from_feature_val = self.edge_attr[from_feature_name][idx] + to_feature_idx = feature_mapping[from_feature_val] + yield to_feature_vals[to_feature_idx] + else: + for pid in pids: + idx = self._edges[pid] + yield self.edge_attr[feature_name][idx] + + def to_triplets(self) -> Iterator[TripletLike]: + return iter(self.edge_attr[EDGE_PID]) + def save(self, path: str) -> None: with open(path, "w") as f: pkl.dump(self, f) - + @classmethod - def from_disk(cls, path: str) -> 'LargeGraphIndexer': + def from_disk(cls, path: str) -> "LargeGraphIndexer": with open(path) as f: - return pkl.load(f) \ No newline at end of file + return pkl.load(f) diff --git a/examples/llm_plus_gnn/raw_qsp_dataset.py b/examples/llm_plus_gnn/raw_qsp_dataset.py new file mode 100644 index 000000000000..1bdb214d7019 --- /dev/null +++ b/examples/llm_plus_gnn/raw_qsp_dataset.py @@ -0,0 +1,103 @@ +from torch_geometric.datasets.web_qsp_dataset import * +from typing import Optional + + +class RawWebQSPDataset(WebQSPDataset): + + def __init__( + self, + root: str = "", + force_reload: bool = False, + with_process: bool = False, + limit: Optional[int] = None, + ) -> None: + self.with_process = with_process + self.limit = limit + if self.with_process: + super().__init__(root, force_reload) + else: + self._check_dependencies() + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + super(InMemoryDataset, self).__init__( + root, None, None, force_reload=force_reload + ) + self._load_raw_data() + + @property + def raw_file_names(self) -> List[str]: + if not self.with_process: + return ["raw_data", "split_idxs"] + else: + return [] + + def _save_raw_data(self) -> None: + self.raw_dataset.save_to_disk(self.raw_paths[0]) + torch.save(self.split_idxs, self.raw_paths[1]) + + def _load_raw_data(self) -> None: + self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) + self.split_idxs = torch.load(self.raw_paths[1]) + + def download(self) -> None: + super().download() + if not self.with_process: + self._save_raw_data() + + def process(self) -> None: + if self.with_process: + pretrained_repo = "sentence-transformers/all-roberta-large-v1" + self.model = SentenceTransformer(pretrained_repo) + self.model.to(self.device) + self.model.eval() + # self.questions = [i["question"] for i in self.raw_dataset] + list_of_graphs = [] + # encode questions + # print("Encoding questions...") + # q_embs = text2embedding(self.model, self.device, self.questions) + print("Encoding graphs...") + limit = self.limit if self.limit else len(self.raw_dataset) + for index in tqdm(range(limit)): + data_i = self.raw_dataset[index] + raw_nodes: Dict[str, int] = {} + raw_edges = [] + for tri in data_i["graph"]: + h, r, t = tri + h = h.lower() + t = t.lower() + if h not in raw_nodes: + raw_nodes[h] = len(raw_nodes) + if t not in raw_nodes: + raw_nodes[t] = len(raw_nodes) + raw_edges.append( + {"src": raw_nodes[h], "edge_attr": r, "dst": raw_nodes[t]} + ) + nodes = pd.DataFrame( + [{"node_id": v, "node_attr": k} for k, v in raw_nodes.items()], + columns=["node_id", "node_attr"], + ) + edges = pd.DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) + # encode nodes + nodes.node_attr = nodes.node_attr.fillna("") + x = text2embedding(self.model, self.device, nodes.node_attr.tolist()) + # encode edges + edge_attr = text2embedding( + self.model, self.device, edges.edge_attr.tolist() + ) + edge_index = torch.LongTensor([edges.src.tolist(), edges.dst.tolist()]) + question = f"Question: {data_i['question']}\nAnswer: " + label = ("|").join(data_i["answer"]).lower() + raw_graph = Data( + x=x, + edge_index=edge_index, + edge_attr=edge_attr, + num_nodes=len(nodes), + ).to("cpu") + list_of_graphs.append(raw_graph) + # psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], + # nodes, edges, topk=3, + # topk_e=5, cost_e=0.5) + # psct_subgraph["question"] = question + # psct_subgraph["label"] = label + # psct_subgraph["desc"] = desc + # list_of_graphs.append(psct_subgraph.to("cpu")) + self.save(list_of_graphs, self.processed_paths[0]) diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_gretriever.ipynb index c6e1212e4c87..0d75cfbdf17e 100644 --- a/examples/llm_plus_gnn/test_eval_gretriever.ipynb +++ b/examples/llm_plus_gnn/test_eval_gretriever.ipynb @@ -15,15 +15,13 @@ } ], "source": [ - "from abc import ABC, abstractmethod\n", - "from typing import Optional, Any, List, Iterable, Tuple, Dict, Union, Set\n", - "from collections.abc import Hashable\n", - "from torch_geometric.data import FeatureStore, InMemoryDataset, GraphStore\n", - "from torch_geometric.data.feature_store import TensorAttr, FeatureTensorType\n", + "from typing import List\n", + "from torch_geometric.data import InMemoryDataset\n", "from torch_geometric.distributed.local_graph_store import LocalGraphStore\n", - "from torch_geometric.datasets.web_qsp_dataset import WebQSPDataset\n", + "from torch_geometric.datasets.web_qsp_dataset import *\n", "import torch\n", - "import datasets" + "import datasets\n", + "import time" ] }, { @@ -32,37 +30,7 @@ "metadata": {}, "outputs": [], "source": [ - "class RawWebQSPDataset(WebQSPDataset):\n", - "\n", - " def __init__(\n", - " self,\n", - " root: str = \"\",\n", - " force_reload: bool = False,\n", - " ) -> None:\n", - " self._check_dependencies()\n", - " self.device = torch.device(\n", - " \"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - " super(InMemoryDataset, self).__init__(root, None, None, force_reload=force_reload)\n", - " self._load_raw_data()\n", - " \n", - " @property\n", - " def raw_file_names(self) -> List[str]:\n", - " return [\"raw_data\", \"split_idxs\"]\n", - "\n", - " def _save_raw_data(self) -> None:\n", - " self.raw_dataset.save_to_disk(self.raw_paths[0])\n", - " torch.save(self.split_idxs, self.raw_paths[1]) \n", - "\n", - " def _load_raw_data(self) -> None:\n", - " self.raw_dataset = datasets.load_from_disk(self.raw_paths[0])\n", - " self.split_idxs = torch.load(self.raw_paths[1])\n", - " \n", - " def download(self) -> None:\n", - " super().download()\n", - " self._save_raw_data()\n", - "\n", - " def process(self) -> None:\n", - " pass" + "from raw_qsp_dataset import RawWebQSPDataset" ] }, { @@ -80,7 +48,7 @@ } ], "source": [ - "dataset = RawWebQSPDataset()" + "dataset = RawWebQSPDataset(force_reload=True)" ] }, { @@ -91,1331 +59,10 @@ { "data": { "text/plain": [ - "[['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", - " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", - " ['Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", - " ['Record producer',\n", - " 'music.performance_role.regular_performances',\n", - " 'm.012m1vf1'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", - " ['2011 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0yrkr34'],\n", - " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Stratford', 'location.location.containedby', 'Canada'],\n", - " ['Singer',\n", - " 'base.lightweight.profession.specialization_of',\n", - " 'Musicians and Singers'],\n", - " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", - " ['As Long As You Love Me (Audien dubstep mix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kevin Risto', 'people.person.gender', 'Male'],\n", - " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", - " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", - " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", - " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", - " ['American Music Award for Favorite Pop/Rock Album',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc259'],\n", - " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", - " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", - " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Billboard Music Award for Top Streaming Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0njhx1b'],\n", - " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", - " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", - " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", - " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.episode',\n", - " 'Results Show: Week 7'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['One Less Lonely Girl',\n", - " 'music.album.primary_release',\n", - " 'One Less Lonely Girl'],\n", - " ['Twista', 'people.person.gender', 'Male'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", - " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", - " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", - " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", - " ['Madonna', 'music.artist.genre', 'Pop music'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", - " ['m.0gbm3cg',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", - " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", - " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", - " ['MTV Video Music Award Japan for Best New Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhrwc'],\n", - " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", - " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", - " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['m.0gctwjk', 'tv.tv_guest_role.episodes_appeared_in', 'Series 2, Episode 3'],\n", - " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", - " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", - " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", - " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", - " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Beauty and a Beat',\n", - " 'music.single.versions',\n", - " 'Beauty and a Beat (Wideboys Club Mix)'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Bryan Adams',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", - " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", - " ['Dany Brillant', 'people.person.gender', 'Male'],\n", - " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", - " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", - " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'music.composition.form', 'Song'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0yrkr34'],\n", - " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", - " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", - " ['Record producer',\n", - " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", - " 'Haley James Scott'],\n", - " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", - " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['Kanye West', 'people.person.profession', 'Singer'],\n", - " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", - " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", - " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", - " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", - " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", - " ['RedOne', 'music.artist.genre', 'Rock music'],\n", - " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", - " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", - " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'music.recording.featured_artists',\n", - " 'Big Sean'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", - " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", - " ['Athan Grace', 'people.person.profession', 'Actor'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", - " ['All Around the World (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", - " ['Brandy Norwood',\n", - " 'influence.influence_node.influenced_by',\n", - " 'Whitney Houston'],\n", - " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['MTV Video Music Award for Artist to Watch',\n", - " 'award.award_category.winners',\n", - " 'm.0n1ykxp'],\n", - " ['Caitlin Beadles',\n", - " 'celebrities.celebrity.sexual_relationships',\n", - " 'm.0d33gyj'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audiobot instrumental)'],\n", - " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", - " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'The Mighty Mighty Bosstones'],\n", - " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", - " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", - " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Documentary film'],\n", - " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", - " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Amerie', 'music.artist.genre', 'Pop music'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", - " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep edit)'],\n", - " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Recovery', 'music.recording.song', 'Recovery'],\n", - " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", - " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", - " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", - " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", - " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", - " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", - " ['#thatPower', 'music.recording.song', '#thatPower'],\n", - " ['Children', 'rdf-schema#range', 'Person'],\n", - " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", - " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['2013 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0wjgqck'],\n", - " ['The Island Def Jam Music Group',\n", - " 'organization.organization.parent',\n", - " 'm.04q65lb'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Rusted Root'],\n", - " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['m.0z87d3n',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", - " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", - " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", - " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", - " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", - " ['Tampa', 'location.location.containedby', 'United States of America'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.compositions',\n", - " 'Love Never Felt So Good'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", - " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", - " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", - " ['Estelle', 'people.person.profession', 'Record producer'],\n", - " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", - " ['Chris Jasper', 'people.person.gender', 'Male'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", - " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", - " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", - " ['Snoop Dogg',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['David Nicksay', 'people.person.gender', 'Male'],\n", - " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", - " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Juno Awards of 2014',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0102z0vx'],\n", - " ['As Long As You Love Me (Audiobot remix)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", - " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", - " ['m.0_cyzs_',\n", - " 'celebrities.legal_entanglement.offense',\n", - " 'Driving under the influence'],\n", - " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", - " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", - " ['Mann', 'people.person.gender', 'Male'],\n", - " ['JoJo', 'people.person.gender', 'Female'],\n", - " ['Right Here (featuring Drake)',\n", - " 'music.recording.canonical_version',\n", - " 'Right Here'],\n", - " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", - " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0yrjynf',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", - " ['Pras', 'people.person.profession', 'Record producer'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", - " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", - " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", - " ['Marvin Isley',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", - " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['School Gyrls', 'film.film.language', 'English Language'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", - " ['Katy Perry', 'people.person.profession', 'Actor'],\n", - " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", - " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.featured_artists',\n", - " 'Redfoo'],\n", - " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", - " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", - " ['As Long as You Love Me (album version)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Justin Bieber: Just Getting Started',\n", - " 'book.written_work.author',\n", - " 'Justin Bieber'],\n", - " ['BeirutNights.com Radio',\n", - " 'broadcast.content.artist',\n", - " 'Marc Maris vs. Ramone'],\n", - " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", - " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", - " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", - " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.releases',\n", - " 'Love Never Felt So Good'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", - " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", - " ['Dance music',\n", - " 'broadcast.genre.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", - " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", - " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", - " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", - " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", - " ['Scooter Braun', 'people.person.gender', 'Male'],\n", - " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", - " ['Sir Nolan', 'people.person.gender', 'Male'],\n", - " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", - " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", - " ['Adam Messinger',\n", - " 'music.composer.compositions',\n", - " \"Turn to You (Mother's Day Dedication)\"],\n", - " ['m.0yrktlv',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Male Hottie'],\n", - " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", - " ['Iggy Azalea',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", - " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['m.0gxnnzy', 'celebrities.romantic_relationship.relationship_type', 'Dated'],\n", - " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", - " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", - " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Britney Spears',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", - " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", - " ['Fergie', 'music.artist.genre', 'Rock music'],\n", - " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", - " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", - " ['Mistletoe', 'music.album.release_type', 'Single'],\n", - " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", - " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.tracks',\n", - " 'Live My Life (Party Rock remix)'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['m.0njw4z2',\n", - " 'award.award_honor.award',\n", - " 'MTV Europe Music Award for Best Male'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", - " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", - " ['m.0gbm3c3',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Miley Cyrus'],\n", - " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", - " ['United States of America',\n", - " 'base.biblioness.bibs_topic.is_really',\n", - " 'United States of America'],\n", - " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", - " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", - " ['JoJo', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Next to You', 'music.album.release_type', 'Single'],\n", - " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", - " ['Willa Ford', 'people.person.languages', 'English Language'],\n", - " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", - " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", - " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", - " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Hot Wired Radio',\n", - " 'broadcast.content.broadcast',\n", - " 'Hot Wired Radio - 128kbps Stream'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", - " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0gbm3d9',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", - " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", - " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", - " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", - " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", - " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", - " ['Justin Timberlake',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Khalil', 'people.person.gender', 'Male'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", - " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", - " ['Selena Gomez',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", - " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", - " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['C1', 'music.artist.genre', 'Hip hop music'],\n", - " ['My Worlds Acoustic',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Album content type'],\n", - " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", - " ['Live My Life', 'music.composition.language', 'English Language'],\n", - " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", - " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", - " ['Baby', 'music.album.releases', 'Baby'],\n", - " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", - " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", - " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", - " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", - " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", - " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", - " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.production_companies',\n", - " 'AEG Live'],\n", - " ['Redfoo', 'people.person.gender', 'Male'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", - " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", - " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", - " ['m.01053qzf',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", - " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", - " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", - " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", - " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", - " ['Ray J', 'people.person.profession', 'Musician'],\n", - " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", - " ['m.0101fv5f',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", - " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", - " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", - " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", - " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.appearance_type',\n", - " 'Guest host'],\n", - " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", - " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", - " ['Christina Milian', 'people.person.profession', 'Actor'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", - " ['Dance music', 'broadcast.genre.content', '181-party'],\n", - " ['Queen Elizabeth II Diamond Jubilee Medal',\n", - " 'award.award_category.winners',\n", - " 'm.0njwqrb'],\n", - " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", - " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", - " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['CMT Music Award: Collaborative Video of the Year',\n", - " 'award.award_category.winners',\n", - " 'm.0njvs9s'],\n", - " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", - " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", - " ['Chingy', 'people.person.profession', 'Actor'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", - " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", - " ['Justin Bieber',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", - " ['Chingy', 'people.person.gender', 'Male'],\n", - " ['Reed Smoot', 'people.person.gender', 'Male'],\n", - " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", - " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.releases',\n", - " 'As Long As You Love Me (remixes)'],\n", - " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjvlh'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", - " ['Singer', 'common.topic.article', 'm.09l6h'],\n", - " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", - " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'award.award_winning_work.awards_won',\n", - " 'm.0pc670l'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", - " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", - " ['Beyoncé Knowles',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['m.0njhyh_',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Streaming Song (Video)'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", - " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_production_design_by',\n", - " 'Devorah Herbert'],\n", - " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", - " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", - " ['Nick Jonas',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", - " ['Lupe Fiasco',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['Martin Kierszenbaum',\n", - " 'people.person.place_of_birth',\n", - " 'United States of America'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long as You Love Me'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", - " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", - " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", - " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", - " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", - " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['m.0y5tl39',\n", - " 'film.personal_film_appearance.film',\n", - " 'Les Coulisses des Golden Globes'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", - " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", - " ['justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z0tgz6'],\n", - " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", - " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", - " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", - " ['Everlast', 'music.artist.label', 'Island Records'],\n", - " ['5th Annual Shorty Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0ywvh8k'],\n", - " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Baby', 'common.topic.notable_types', 'Composition'],\n", - " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['m.0tkqqgg',\n", - " 'award.award_nomination.award',\n", - " 'Juno Award for Pop Album of the Year'],\n", - " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", - " ['Person', 'type.type.properties', 'Parents'],\n", - " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Nasri', 'people.person.profession', 'Singer'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.compositions',\n", - " 'As Long as You Love Me'],\n", - " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", - " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", - " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", - " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", - " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", - " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", - " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", - " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", - " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", - " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", - " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.film',\n", - " 'Zendaya: Behind the Scenes'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", - " ['That Should Be Me', 'music.composition.form', 'Song'],\n", - " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", - " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Justin Bieber'],\n", - " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.song',\n", - " 'Beauty And A Beat'],\n", - " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", - " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Usher', 'music.composer.compositions', 'First Dance'],\n", - " ['m.0n1ykxp',\n", - " 'award.award_honor.award',\n", - " 'MTV Video Music Award for Artist to Watch'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Rockumentary'],\n", - " ['Amerie', 'people.person.gender', 'Female'],\n", - " ['Real Change: Artists for Education',\n", - " 'film.film.personal_appearances',\n", - " 'm.0y5th3r'],\n", - " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", - " ['Beautiful and the Beat',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['PYD', 'common.topic.notable_types', 'Composition'],\n", - " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", - " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", - " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", - " ['iJustine', 'people.person.gender', 'Female'],\n", - " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", - " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", - " ['m.0ng_vkd',\n", - " 'film.personal_film_appearance.film',\n", - " 'A Michael Bublé Christmas'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", - " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", - " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", - " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Vanessa Hudgens',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", - " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", - " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Madonna', 'people.person.profession', 'Record producer'],\n", - " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", - " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", - " ['As Long as You Love Me (acoustic version)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", - " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", - " ['Rob Thomas', 'people.person.gender', 'Male'],\n", - " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", - " ['m.0hvlt03',\n", - " 'film.film_film_distributor_relationship.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", - " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_srv2b'],\n", - " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", - " ['Donna Summer',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['m.0101fszs',\n", - " 'film.personal_film_appearance.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Alanis Morissette',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Official website'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Jenna Andrews'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", - " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Ray J', 'people.person.nationality', 'United States of America'],\n", - " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Nelly Furtado',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['m.0gbm3fj',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", - " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", - " ['As Long As You Love Me (Ferry Corsten club dub)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", - " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", - " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", - " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", - " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", - " ['Guest host',\n", - " 'tv.non_character_role.tv_guest_personal_appearances',\n", - " 'm.0v_98y7'],\n", - " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", - " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", - " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", - " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", - " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", - " ['L.A. Reid', 'people.person.gender', 'Male'],\n", - " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['London', 'location.location.containedby', 'Ontario'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_set_decoration_by',\n", - " 'Lia Roldan'],\n", - " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", - " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", - " ['Children', 'type.property.schema', 'Person'],\n", - " ['Change Me', 'music.album.releases', 'Change Me'],\n", - " ['RedOne', 'music.artist.label', 'Island Records'],\n", - " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", - " ['All Around the World',\n", - " 'music.recording.canonical_version',\n", - " 'All Around the World'],\n", - " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0wjhc6c'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", - " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", - " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Lupe Fiasco'],\n", - " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", - " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", - " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", - " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", - " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Beauty and a Beat (Wideboys Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", - " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", - " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", - " ['m.0v90skf',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Male Artist'],\n", - " ['Ludacris', 'people.person.profession', 'Actor'],\n", - " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", - " ['Cameo appearance',\n", - " 'tv.special_tv_performance_type.episode_performances',\n", - " 'm.0v1lwt2'],\n", - " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", - " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", - " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'One Chance'],\n", - " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", - " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", - " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", - " ['Roller Coaster',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0_x4zg3'],\n", - " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", - " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", - " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", - " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", - " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", - " ['m.0sgk_cw',\n", - " 'award.award_honor.award',\n", - " \"Kids' Choice Award for Favorite Song\"],\n", - " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", - " ['MTV Video Music Brazil Award for Best International Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhhqv'],\n", - " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", - " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", - " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", - " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", - " ['Britney Spears', 'people.person.profession', 'Singer'],\n", - " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", - " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Rock music',\n", - " 'base.webvideo.internet_video_genre.series',\n", - " 'Biscuithands, The Animated Musical'],\n", - " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", - " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Barry Weiss'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['MTV Europe Music Award for Best Male',\n", - " 'award.award_category.winners',\n", - " 'm.0z1scxk'],\n", - " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", - " ['Will Smith', 'people.person.profession', 'Actor'],\n", - " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", - " ['Will i Am', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Fabian', 'people.person.gender', 'Male'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", - " ['Somebody to Love (remix)',\n", - " 'music.album.primary_release',\n", - " 'Somebody to Love (remix)'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", - " ['Spouse', 'type.property.expected_type', 'Person'],\n", - " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", - " ['Baby', 'music.recording.artist', 'Ludacris'],\n", - " ['Rudolph Valentino',\n", - " 'people.person.nationality',\n", - " 'United States of America'],\n", - " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", - " ['Judy Garland',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Kelly Clarkson',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'base.schemastaging.context_name.pronunciation',\n", - " 'm.011h9_22'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", - " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", - " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0vp8rhw',\n", - " 'music.recording_contribution.album',\n", - " 'Beauty and a Beat (Remixes)'],\n", - " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", - " ['Katy Perry: Part of Me',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['m.0jsmvv5',\n", - " 'film.film_regional_release_date.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", - " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", - " ['All Around The World (featuring Ludacris)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", - " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", - " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", - " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", - " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", - " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep mix)'],\n", - " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Fergie', 'people.person.profession', 'Actor'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", - " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", - " ['Zac Efron', 'people.person.gender', 'Male'],\n", - " ['P!nk', 'music.artist.genre', 'Rock music'],\n", - " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", - " ['Gender', 'type.property.schema', 'Person'],\n", - " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", - " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", - " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", - " ['Under the Mistletoe',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Initial release date'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Slick Rick'],\n", - " ['Amerie', 'music.artist.genre', 'Rock music'],\n", - " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0pbzq13',\n", - " 'film.performance.special_performance_type',\n", - " 'Cameo appearance'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", - " ['Height', 'type.property.unit', 'Meter'],\n", - " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", - " ['Ray J',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Boyfriend (acoustic version)',\n", - " 'music.recording.canonical_version',\n", - " 'Boyfriend'],\n", - " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Believe Tour', 'music.concert_tour.album_or_release_supporting', 'Believe'],\n", - " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", - " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", - " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", - " ['Frank Ocean',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['As Long As You Love Me (Audiobot instrumental)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Usher', 'people.person.nationality', 'United States of America'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", - " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", - " ['m.0gbm3b7',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Sheryl Crow',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['All That Matters',\n", - " 'music.composition.composer',\n", - " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Nasri', 'music.artist.genre', 'Reggae'],\n", - " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", - " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", - " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", - " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", - " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", - " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", - " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long As You Love Me'],\n", - " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Bryan-Michael Cox',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", - " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", - " ['Whitney Houston', 'people.person.profession', 'Model'],\n", - " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['As Long as You Love Me',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Disney Parks Christmas Day Parade',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['Ray J', 'people.person.profession', 'Artist'],\n", - " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", - " ['American Music Award for Favorite Pop/Rock Male Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc0sf'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", - " ['The Notorious B.I.G.',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", - " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", - " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", - " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", - " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Haley James Scott',\n", - " 'fictional_universe.fictional_character.occupation',\n", - " 'Record producer'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", - " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", - " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['My Worlds: The Collection',\n", - " 'music.album.album_content_type',\n", - " 'Compilation album'],\n", - " ['Lolly', 'music.album.releases', 'Lolly'],\n", - " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", - " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", - " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", - " ['Ja Rule', 'people.person.profession', 'Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", - " ['Die in Your Arms',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0z85qxq'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", - " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", - " ['Kuk Harrell',\n", - " 'film.person_or_entity_appearing_in_film.films',\n", - " 'm.0101ft5f'],\n", - " ['Somebody to Love (J Stax remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'film.film.executive_produced_by',\n", - " 'Allison Kaye Scarinzi'],\n", - " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", - " ['Nasri', 'music.artist.genre', 'Pop music'],\n", - " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", - " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", - " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", - " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", - " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.05sp405',\n", - " 'organization.organization_relationship.child',\n", - " 'Island Records'],\n", - " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", - " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", - " ['John Mamann',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Teen Choice Award for Choice Summer Music Star: Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjynf'],\n", - " ['Juicy J', 'people.person.profession', 'Actor'],\n", - " ['m.0yqflrk',\n", - " 'measurement_unit.dated_money_value.source',\n", - " 'celebritynetworth.com'],\n", - " ['Miley Cyrus',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", - " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['m.04q65lb',\n", - " 'organization.organization_relationship.child',\n", - " 'The Island Def Jam Music Group'],\n", - " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", - " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", - " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['My World', 'music.album.artist', 'Justin Bieber'],\n", - " ['Don Henley', 'people.person.gender', 'Male'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Musician', 'people.profession.specializations', 'Singer'],\n", - " ['Die in Your Arms', 'music.recording.canonical_version', 'Die in Your Arms'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['m.0njvs9s',\n", - " 'award.award_honor.award',\n", - " 'CMT Music Award: Collaborative Video of the Year'],\n", - " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber',\n", - " 'music.artist.album',\n", - " 'Turn to You (Mother’s Day Dedication)'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", - " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['City/Town/Village', 'freebase.type_profile.strict_included_types', 'Topic'],\n", - " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", - " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Terence Dudley', 'people.person.gender', 'Male'],\n", - " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Mr. Sosa & The Yayo'],\n", - " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", - " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", - " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Gender', 'type.property.expected_type', 'Gender'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0101fvbf',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['m.0yrhrwc', 'award.award_honor.ceremony', '2011 MTV Video Music Aid Japan'],\n", - " ['MTV Europe Music Award for Best North American Act',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhmll'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['m.0101ft1d',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", - " ['Ciara', 'people.person.gender', 'Female'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", - " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", - " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0jzrrqs', 'location.mailing_address.country', 'United States of America'],\n", - " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", - " ['Big R Radio - Top 40 Hits',\n", - " 'broadcast.content.artist',\n", - " 'Natasha Bedingfield'],\n", - " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ...]" + "Dataset({\n", + " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", + " num_rows: 4700\n", + "})" ] }, "execution_count": 4, @@ -1424,71 +71,23 @@ } ], "source": [ - "dataset.raw_dataset[0]['graph']" + "dataset.raw_dataset" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "test_graph_store = LocalGraphStore()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attr = dict(\n", - " edge_type=None,\n", - " layout='coo',\n", - " size=(2,2),\n", - " is_sorted=False\n", - ")\n", - "test_graph_store.put_edge_index(torch.Tensor([[0,1], [1,2]]), **attr)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[EdgeAttr(edge_type=None, layout=, is_sorted=False, size=(2, 2))]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "test_graph_store.get_all_edge_attrs()" + "from large_graph_indexer import LargeGraphIndexer" ] }, { - "cell_type": "code", - "execution_count": 8, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from large_graph_indexer import LargeGraphIndexer" + "Test basic collation" ] }, { @@ -1524,8 +123,8 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(indexer.nodes) + len(indexer2.nodes) - len(indexer.nodes.keys() & indexer2.nodes.keys()) == len(bigger_indexer.nodes)\n", - "assert len(indexer.edges) + len(indexer2.edges) - len(indexer.edges.keys() & indexer2.edges.keys()) == len(bigger_indexer.edges)" + "assert len(indexer._nodes) + len(indexer2._nodes) - len(indexer._nodes.keys() & indexer2._nodes.keys()) == len(bigger_indexer._nodes)\n", + "assert len(indexer._edges) + len(indexer2._edges) - len(indexer._edges.keys() & indexer2._edges.keys()) == len(bigger_indexer._edges)" ] }, { @@ -1534,8 +133,8 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(set(bigger_indexer.nodes.values())) == len(bigger_indexer.nodes)\n", - "assert len(set(bigger_indexer.edges.values())) == len(bigger_indexer.edges)" + "assert len(set(bigger_indexer._nodes.values())) == len(bigger_indexer._nodes)\n", + "assert len(set(bigger_indexer._edges.values())) == len(bigger_indexer._edges)" ] }, { @@ -1544,8 +143,8 @@ "metadata": {}, "outputs": [], "source": [ - "for node in indexer.nodes.keys():\n", - " assert indexer.node_attr[indexer.nodes[node]][\"pid\"] == node, f'{node} is not {indexer.node_attr[indexer.nodes[node]][\"pid\"]}'" + "for node in indexer._nodes.keys():\n", + " assert indexer.node_attr['pid'][indexer._nodes[node]] == node, f'{node} is not {indexer.node_attr[\"pid\"][indexer._nodes[node]]}'" ] }, { @@ -1558,61 +157,76 @@ "from multiprocessing import Pool" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Test collation on entire dataset" + ] + }, { "cell_type": "code", "execution_count": 16, "metadata": {}, + "outputs": [], + "source": [ + "def get_next_graph(dataset):\n", + " for ds in dataset:\n", + " yield ds['graph']\n", + "graphs = get_next_graph(dataset.raw_dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, "outputs": [ { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#TODO: can probably be parallelized\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Pool(\u001b[38;5;241m40\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m p:\n\u001b[0;32m----> 4\u001b[0m indexers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(tqdm\u001b[38;5;241m.\u001b[39mtqdm(p\u001b[38;5;241m.\u001b[39mimap(LargeGraphIndexer\u001b[38;5;241m.\u001b[39mfrom_triplets, [ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgraph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m dataset\u001b[38;5;241m.\u001b[39mraw_dataset]), total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset\u001b[38;5;241m.\u001b[39mraw_dataset)))\n", - "Cell \u001b[0;32mIn[16], line 4\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#TODO: can probably be parallelized\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Pool(\u001b[38;5;241m40\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m p:\n\u001b[0;32m----> 4\u001b[0m indexers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(tqdm\u001b[38;5;241m.\u001b[39mtqdm(p\u001b[38;5;241m.\u001b[39mimap(LargeGraphIndexer\u001b[38;5;241m.\u001b[39mfrom_triplets, [ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgraph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m dataset\u001b[38;5;241m.\u001b[39mraw_dataset]), total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset\u001b[38;5;241m.\u001b[39mraw_dataset)))\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/arrow_dataset.py:2440\u001b[0m, in \u001b[0;36mDataset.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 2438\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(pa_subtable\u001b[38;5;241m.\u001b[39mnum_rows):\n\u001b[1;32m 2439\u001b[0m pa_subtable_ex \u001b[38;5;241m=\u001b[39m pa_subtable\u001b[38;5;241m.\u001b[39mslice(i, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m-> 2440\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformat_table\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2441\u001b[0m \u001b[43m \u001b[49m\u001b[43mpa_subtable_ex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2442\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2443\u001b[0m \u001b[43m \u001b[49m\u001b[43mformatter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformatter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2444\u001b[0m \u001b[43m \u001b[49m\u001b[43mformat_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_format_columns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2445\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_all_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_output_all_columns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2446\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2447\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m formatted_output\n\u001b[1;32m 2448\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:641\u001b[0m, in \u001b[0;36mformat_table\u001b[0;34m(table, key, formatter, format_columns, output_all_columns)\u001b[0m\n\u001b[1;32m 639\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 640\u001b[0m pa_table_to_format \u001b[38;5;241m=\u001b[39m pa_table\u001b[38;5;241m.\u001b[39mdrop(col \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m pa_table\u001b[38;5;241m.\u001b[39mcolumn_names \u001b[38;5;28;01mif\u001b[39;00m col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m format_columns)\n\u001b[0;32m--> 641\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformatter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table_to_format\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_type\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 642\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_all_columns:\n\u001b[1;32m 643\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(formatted_output, MutableMapping):\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:397\u001b[0m, in \u001b[0;36mFormatter.__call__\u001b[0;34m(self, pa_table, query_type)\u001b[0m\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable, query_type: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[RowFormat, ColumnFormat, BatchFormat]:\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrow\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 397\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat_column(pa_table)\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:437\u001b[0m, in \u001b[0;36mPythonFormatter.format_row\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlazy:\n\u001b[1;32m 436\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m LazyRow(pa_table, \u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m--> 437\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpython_arrow_extractor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 438\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpython_features_decoder\u001b[38;5;241m.\u001b[39mdecode_row(row)\n\u001b[1;32m 439\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m row\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/datasets/formatting/formatting.py:145\u001b[0m, in \u001b[0;36mPythonArrowExtractor.extract_row\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mextract_row\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[0;32m--> 145\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _unnest(\u001b[43mpa_table\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_pydict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 4700/4700 [02:57<00:00, 26.41it/s]\n" ] } ], "source": [ "indexers = []\n", - "#TODO: can probably be parallelized\n", "with Pool(40) as p:\n", - " indexers = list(tqdm.tqdm(p.imap(LargeGraphIndexer.from_triplets, [ds['graph'] for ds in dataset.raw_dataset]), total=len(dataset.raw_dataset)))" + " indexers = list(tqdm.tqdm(p.imap(LargeGraphIndexer.from_triplets, graphs), total=len(dataset.raw_dataset)))" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "metadata": {}, "outputs": [ { - "ename": "NameError", - "evalue": "name 'indexers' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#FIXME: right now this is really slow\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m big_indexer \u001b[38;5;241m=\u001b[39m LargeGraphIndexer\u001b[38;5;241m.\u001b[39mcollate(tqdm\u001b[38;5;241m.\u001b[39mtqdm(\u001b[43mindexers\u001b[49m), skip_shared_check\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", - "\u001b[0;31mNameError\u001b[0m: name 'indexers' is not defined" - ] + "data": { + "text/plain": [ + "15.804845809936523" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#FIXME: right now this is really slow\n", - "big_indexer = LargeGraphIndexer.collate(tqdm.tqdm(indexers), skip_shared_check=True)" + "start = time.time()\n", + "big_indexer = LargeGraphIndexer.collate(indexers)\n", + "time.time()-start" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Naive method, concatenating all the triplets" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -1621,42 +235,35 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "large_graph_dataset = chain.from_iterable([ds['graph'] for ds in dataset.raw_dataset])" + "large_graph_dataset = chain.from_iterable(get_next_graph(dataset.raw_dataset))" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 28, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 4700/4700 [00:51<00:00, 92.08it/s] \n" - ] - } - ], + "outputs": [], "source": [ + "# This is just for TQDM to work well\n", "total_size = 0\n", - "for ds in tqdm.tqdm(dataset.raw_dataset):\n", - " total_size += len(ds['graph'])" + "#for g in tqdm.tqdm(get_next_graph(dataset.raw_dataset), total=4700):\n", + "# total_size += len(g)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 19986134/19986134 [00:20<00:00, 956393.67it/s] \n" + "19986134it [04:28, 74537.24it/s] \n" ] } ], @@ -1666,27 +273,64 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ - "node_attributes = list(large_indexer.get_unique_node_features(\"pid\"))\n", + "assert set(large_indexer._nodes) == set(big_indexer._nodes)\n", + "assert set(large_indexer._edges) == set(big_indexer._edges)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Phase I: Indexing Large Graph" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "node_attributes = list(big_indexer.get_unique_node_features(\"pid\"))\n", "node_attributes = [i.lower() for i in node_attributes]" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ - "edge_attributes = list(large_indexer.get_unique_edge_features(\"r\"))\n", + "edge_attributes = list(big_indexer.get_unique_edge_features(\"r\"))\n", "edge_attributes = [i.lower() for i in edge_attributes]" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1298306" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(node_attributes)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -1695,7 +339,7 @@ "6094" ] }, - "execution_count": 23, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1706,7 +350,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1715,7 +359,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1773,7 +417,7 @@ ")" ] }, - "execution_count": 25, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -1786,7 +430,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -1803,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1812,171 +456,420 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "nodes_to_process, edges_to_process = len(node_attributes), len(edge_attributes)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 32" + ] + }, + { + "cell_type": "code", + "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 0%| | 0/5071 [00:00 Data:\n", + " nodes = {}\n", + " edges = []\n", + " edge_index = []\n", "\n", - "#node_embs = text2embedding(model, device, node_attributes)\n", - "#edge_embs = text2embedding(model, device, edge_attributes)\n", + " for h, r, t in triplets:\n", + " if h not in nodes:\n", + " nodes[h] = len(nodes)\n", + " if t not in nodes:\n", + " nodes[t] = len(nodes)\n", + " \n", + " edges.append((h, r, t))\n", + " edge_index.append([nodes[h], nodes[t]])\n", + "\n", + " node_keys = list(nodes.keys())\n", + " edge_keys = ordered_set(edges)\n", + " \n", + " node_feats: Iterable[FeatureTensorType] = indexer.get_node_features(feature_name=node_feature_name, pids=node_keys)\n", + " edge_feats: Iterable[FeatureTensorType] = indexer.get_edge_features(feature_name=edge_feature_name, pids=edge_keys)\n", + "\n", + " x=torch.stack(node_feats, 0)\n", + " edge_attr=torch.stack(edge_feats, 0)\n", + " edge_index=torch.t(torch.LongTensor(edge_index))\n", + " return Data(x=x, edge_index=edge_index,edge_attr=edge_attr,num_nodes=len(nodes))\n", "\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "first_10_trips = dataset.raw_dataset[:10]['graph']" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/10 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from_=85\n", + "to=90\n", + "plt.matshow((first_10[0].x[from_:to,:100]-old_dataset[0].x[from_:to,:100]).abs())\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male']]" + ] + }, + "execution_count": 102, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.raw_dataset[0]['graph'][85:90]" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male']]" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "old_dataset.raw_dataset[0]['graph'][85:90]" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "9088" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(old_dataset.raw_dataset[0]['graph'])" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "9088" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(dataset.raw_dataset[0]['graph'])" ] }, { From b9126522bd94b5a2bde25a0cfeb4dd6e58e3c462 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 29 May 2024 20:14:57 -0700 Subject: [PATCH 584/752] finished dataloader --- examples/llm_plus_gnn/large_graph_indexer.py | 165 ++++- .../llm_plus_gnn/test_eval_gretriever.ipynb | 582 ++++++++++-------- 2 files changed, 471 insertions(+), 276 deletions(-) diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/examples/llm_plus_gnn/large_graph_indexer.py index 4f8c700d4311..b55d551ed707 100644 --- a/examples/llm_plus_gnn/large_graph_indexer.py +++ b/examples/llm_plus_gnn/large_graph_indexer.py @@ -1,18 +1,23 @@ +import os +import pickle as pkl +from dataclasses import dataclass +from itertools import chain from typing import ( - Tuple, - Hashable, + Any, + Callable, Dict, + Hashable, Iterable, - Set, - Optional, - Any, Iterator, List, + Optional, Sequence, + Set, + Tuple, ) -import pickle as pkl -from dataclasses import dataclass -from itertools import chain + +import torch +from torch_geometric.data import Data # Is there a multiprocessing-friendly implementation of this? @@ -36,6 +41,14 @@ class MappedFeature: name: str values: Sequence[Any] + def __eq__(self, value: "MappedFeature") -> bool: + eq = self.name == value.name + if isinstance(self.values, torch.Tensor): + eq &= torch.equal(self.values, value.values) + else: + eq &= self.values == value.values + return eq + class LargeGraphIndexer: """For a dataset that consists of mulitiple subgraphs that are assumed to @@ -90,11 +103,23 @@ def __init__( self._edges[tup] = i @classmethod - def from_triplets(cls, triplets: Iterable[TripletLike]) -> "LargeGraphIndexer": + def from_triplets( + cls, + triplets: Iterable[TripletLike], + pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, + ) -> "LargeGraphIndexer": # NOTE: Right now assumes that all trips can be loaded into memory nodes = set() edges = set() + if pre_transform is not None: + + def apply_transform(trips: Iterable[TripletLike]) -> Iterator[TripletLike]: + for trip in trips: + yield pre_transform(trip) + + triplets = apply_transform(triplets) + for h, r, t in triplets: for node in (h, t): @@ -212,14 +237,14 @@ def add_edge_feature( def get_edge_features( self, - feature_name: str = EDGE_RELATION, + feature_name: str = EDGE_PID, pids: Optional[Iterable[Hashable]] = None, ) -> List[Any]: return list(self.get_edge_features_iter(feature_name, pids)) def get_edge_features_iter( self, - feature_name: str = EDGE_RELATION, + feature_name: str = EDGE_PID, pids: Optional[Iterable[TripletLike]] = None, ) -> Iterator[Any]: if pids is None: @@ -248,10 +273,120 @@ def to_triplets(self) -> Iterator[TripletLike]: return iter(self.edge_attr[EDGE_PID]) def save(self, path: str) -> None: - with open(path, "w") as f: - pkl.dump(self, f) + os.makedirs(path, exist_ok=True) + with open(path + "/edges", "wb") as f: + pkl.dump(self._edges, f) + with open(path + "/nodes", "wb") as f: + pkl.dump(self._nodes, f) + + with open(path + "/mapped_edges", "wb") as f: + pkl.dump(self._mapped_edge_features, f) + with open(path + "/mapped_nodes", "wb") as f: + pkl.dump(self._mapped_node_features, f) + + node_attr_path = path + "/node_attr" + os.makedirs(node_attr_path, exist_ok=True) + for attr_name, vals in self.node_attr.items(): + torch.save(vals, node_attr_path + f"/{attr_name}.pt") + + edge_attr_path = path + "/edge_attr" + os.makedirs(edge_attr_path, exist_ok=True) + for attr_name, vals in self.edge_attr.items(): + torch.save(vals, edge_attr_path + f"/{attr_name}.pt") @classmethod def from_disk(cls, path: str) -> "LargeGraphIndexer": - with open(path) as f: - return pkl.load(f) + indexer = cls(list(), list()) + with open(path + "/edges", "rb") as f: + indexer._edges = pkl.load(f) + with open(path + "/nodes", "rb") as f: + indexer._nodes = pkl.load(f) + + with open(path + "/mapped_edges", "rb") as f: + indexer._mapped_edge_features = pkl.load(f) + with open(path + "/mapped_nodes", "rb") as f: + indexer._mapped_node_features = pkl.load(f) + + node_attr_path = path + "/node_attr" + for fname in os.listdir(node_attr_path): + full_fname = f"{node_attr_path}/{fname}" + key = fname.split(".")[0] + indexer.node_attr[key] = torch.load(full_fname) + + edge_attr_path = path + "/edge_attr" + for fname in os.listdir(edge_attr_path): + full_fname = f"{edge_attr_path}/{fname}" + key = fname.split(".")[0] + indexer.edge_attr[key] = torch.load(full_fname) + + return indexer + + def __eq__(self, value: "LargeGraphIndexer") -> bool: + eq = True + eq &= self._nodes == value._nodes + eq &= self._edges == value._edges + eq &= self.node_attr.keys() == value.node_attr.keys() + eq &= self.edge_attr.keys() == value.edge_attr.keys() + eq &= self._mapped_node_features == value._mapped_node_features + eq &= self._mapped_edge_features == value._mapped_edge_features + + for k in self.node_attr: + eq &= type(self.node_attr[k]) == type(value.node_attr[k]) + if isinstance(self.node_attr[k], torch.Tensor): + eq &= torch.equal(self.node_attr[k], value.node_attr[k]) + else: + eq &= self.node_attr[k] == value.node_attr[k] + for k in self.edge_attr: + eq &= type(self.edge_attr[k]) == type(value.edge_attr[k]) + if isinstance(self.edge_attr[k], torch.Tensor): + eq &= torch.equal(self.edge_attr[k], value.edge_attr[k]) + else: + eq &= self.edge_attr[k] == value.edge_attr[k] + return eq + + +def get_features_for_triplets( + indexer: LargeGraphIndexer, + triplets: Iterable[TripletLike], + node_feature_name: str = "embs", + edge_feature_name: str = "embs", + pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, +) -> Data: + + if pre_transform is not None: + + def apply_transform(trips): + for trip in trips: + yield pre_transform(tuple(trip)) + + # TODO: Make this safe for large amounts of triplets? + triplets = list(apply_transform(triplets)) + + small_graph_indexer = LargeGraphIndexer.from_triplets( + triplets, pre_transform=pre_transform + ) + node_keys = small_graph_indexer.get_node_features() + node_feats = indexer.get_node_features( + feature_name=node_feature_name, pids=node_keys + ) + + edge_keys = small_graph_indexer.get_edge_features(pids=triplets) + edge_feats = indexer.get_edge_features( + feature_name=edge_feature_name, pids=edge_keys + ) + + edge_index = [] + for h, t in zip( + small_graph_indexer.get_edge_features_iter("h", triplets), + small_graph_indexer.get_edge_features_iter("t", triplets), + ): + edge_index.append( + [small_graph_indexer._nodes[h], small_graph_indexer._nodes[t]] + ) + + x = torch.stack(node_feats, 0) + edge_attr = torch.stack(edge_feats, 0) + edge_index = torch.t(torch.LongTensor(edge_index)) + return Data( + x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(node_feats) + ) diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_gretriever.ipynb index 0d75cfbdf17e..7f805250a5b7 100644 --- a/examples/llm_plus_gnn/test_eval_gretriever.ipynb +++ b/examples/llm_plus_gnn/test_eval_gretriever.ipynb @@ -2,18 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 48, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "from typing import List\n", "from torch_geometric.data import InMemoryDataset\n", @@ -26,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -35,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 50, "metadata": {}, "outputs": [ { @@ -53,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 51, "metadata": {}, "outputs": [ { @@ -65,7 +56,7 @@ "})" ] }, - "execution_count": 4, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -76,11 +67,21 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ - "from large_graph_indexer import LargeGraphIndexer" + "from large_graph_indexer import LargeGraphIndexer, TripletLike" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess_triplet(triplet: TripletLike):\n", + " return tuple([word.lower() for word in triplet])" ] }, { @@ -92,25 +93,25 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ - "indexer = LargeGraphIndexer.from_triplets(dataset.raw_dataset[0]['graph'])" + "indexer = LargeGraphIndexer.from_triplets(dataset.raw_dataset[0]['graph'], pre_transform=preprocess_triplet)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ - "indexer2 = LargeGraphIndexer.from_triplets(dataset.raw_dataset[1]['graph'])" + "indexer2 = LargeGraphIndexer.from_triplets(dataset.raw_dataset[1]['graph'], pre_transform=preprocess_triplet)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -119,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -139,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -149,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -166,47 +167,214 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ - "def get_next_graph(dataset):\n", + "LIMIT=100\n", + "def get_next_graph(dataset, limit=None):\n", + " i = 0\n", " for ds in dataset:\n", + " if i == limit:\n", + " break\n", " yield ds['graph']\n", - "graphs = get_next_graph(dataset.raw_dataset)" + " i += 1\n", + "graphs = get_next_graph(dataset.raw_dataset, limit=LIMIT)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 62, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4700/4700 [02:57<00:00, 26.41it/s]\n" + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "100%|██████████| 100/100 [00:02<00:00, 38.64it/s]\n" ] } ], "source": [ "indexers = []\n", + "def from_trips_with_pretransform(triplets):\n", + " return LargeGraphIndexer.from_triplets(triplets, pre_transform=preprocess_triplet)\n", "with Pool(40) as p:\n", - " indexers = list(tqdm.tqdm(p.imap(LargeGraphIndexer.from_triplets, graphs), total=len(dataset.raw_dataset)))" + " indexers = list(tqdm.tqdm(p.imap(from_trips_with_pretransform, graphs), total=len(dataset.raw_dataset) if LIMIT is None else LIMIT))" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "15.804845809936523" + "0.5566630363464355" ] }, - "execution_count": 18, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -226,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -235,16 +403,16 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ - "large_graph_dataset = chain.from_iterable(get_next_graph(dataset.raw_dataset))" + "large_graph_dataset = chain.from_iterable(get_next_graph(dataset.raw_dataset, limit=LIMIT))" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -256,24 +424,24 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 67, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "19986134it [04:28, 74537.24it/s] \n" + "423763it [00:02, 198983.48it/s]\n" ] } ], "source": [ - "large_indexer = LargeGraphIndexer.from_triplets(tqdm.tqdm(large_graph_dataset, total=total_size))" + "large_indexer = LargeGraphIndexer.from_triplets(tqdm.tqdm(large_graph_dataset, total=total_size), pre_transform=preprocess_triplet)" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -290,36 +458,34 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ - "node_attributes = list(big_indexer.get_unique_node_features(\"pid\"))\n", - "node_attributes = [i.lower() for i in node_attributes]" + "node_attributes = list(big_indexer.get_unique_node_features())" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ - "edge_attributes = list(big_indexer.get_unique_edge_features(\"r\"))\n", - "edge_attributes = [i.lower() for i in edge_attributes]" + "edge_attributes = list(big_indexer.get_unique_edge_features(\"r\"))" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 71, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "1298306" + "105413" ] }, - "execution_count": 33, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -330,16 +496,16 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 72, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "6094" + "3095" ] }, - "execution_count": 34, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -350,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -359,7 +525,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 74, "metadata": {}, "outputs": [ { @@ -417,7 +583,7 @@ ")" ] }, - "execution_count": 36, + "execution_count": 74, "metadata": {}, "output_type": "execute_result" } @@ -430,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 75, "metadata": {}, "outputs": [ { @@ -447,7 +613,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -456,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -465,7 +631,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -474,14 +640,14 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 79, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "40573it [14:15, 47.44it/s] \n" + "3295it [01:08, 47.83it/s] \n" ] } ], @@ -496,16 +662,16 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "torch.Size([1298306, 1024])" + "torch.Size([105413, 1024])" ] }, - "execution_count": 42, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -516,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -525,11 +691,11 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ - "from typing import Iterable\n", + "from typing import Iterable, Callable, Optional\n", "from large_graph_indexer import TripletLike, ordered_set\n", "from torch_geometric.typing import FeatureTensorType\n", "from torch_geometric.data import Data" @@ -537,14 +703,14 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 83, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "191it [00:03, 50.03it/s] \n" + "97it [00:01, 49.56it/s] \n" ] } ], @@ -558,16 +724,16 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 84, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "torch.Size([6094, 1024])" + "torch.Size([3095, 1024])" ] }, - "execution_count": 46, + "execution_count": 84, "metadata": {}, "output_type": "execute_result" } @@ -578,7 +744,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -587,40 +753,25 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def get_features_for_triplets(indexer: LargeGraphIndexer, triplets: Iterable[TripletLike], node_feature_name: str = \"embs\", edge_feature_name: str = \"embs\") -> Data:\n", - " nodes = {}\n", - " edges = []\n", - " edge_index = []\n", - "\n", - " for h, r, t in triplets:\n", - " if h not in nodes:\n", - " nodes[h] = len(nodes)\n", - " if t not in nodes:\n", - " nodes[t] = len(nodes)\n", - " \n", - " edges.append((h, r, t))\n", - " edge_index.append([nodes[h], nodes[t]])\n", - "\n", - " node_keys = list(nodes.keys())\n", - " edge_keys = ordered_set(edges)\n", - " \n", - " node_feats: Iterable[FeatureTensorType] = indexer.get_node_features(feature_name=node_feature_name, pids=node_keys)\n", - " edge_feats: Iterable[FeatureTensorType] = indexer.get_edge_features(feature_name=edge_feature_name, pids=edge_keys)\n", - "\n", - " x=torch.stack(node_feats, 0)\n", - " edge_attr=torch.stack(edge_feats, 0)\n", - " edge_index=torch.t(torch.LongTensor(edge_index))\n", - " return Data(x=x, edge_index=edge_index,edge_attr=edge_attr,num_nodes=len(nodes))\n", - "\n" + "from large_graph_indexer import get_features_for_triplets" ] }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "import networkx as nx" + ] + }, + { + "cell_type": "code", + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -629,32 +780,25 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 88, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 0%| | 0/10 [00:00 thresh)\n", + " def _graphs_are_same(tensor1, tensor2):\n", + " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", + " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", + " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", + " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" ] }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 91, "metadata": {}, "outputs": [ { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABP4AAABuCAYAAAC3Kd2QAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUuElEQVR4nO3df3BU9bnH8c9uQpaAm5WACaRJai7XFpDfBGjE62BJ5XKVkSu2eIe2KTr9MS6UkNaW9BZo64+gTi0jv7W2/FEzUDuTUp2Kw00lSAc0BPFClaCVXlYxBLyyCwE2ye65/9StuRLNs8lmk837NbMDe/Y5+T67e55zzj5zfrgcx3EEAAAAAAAAIKW4k50AAAAAAAAAgJ5H4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQX2i8bdx40Zde+21Gjx4sGbOnKlXXnkl2SkBMKqqqtL06dPl9XqVk5OjBQsWqLGxsUPM5cuX5ff7NXz4cF111VVauHChTp8+naSMAcRr7dq1crlcKi8vj02jvoH+691339VXv/pVDR8+XJmZmZowYYIOHjwYe91xHK1evVqjRo1SZmamSktL9eabbyYxYwBdEYlEtGrVKhUVFSkzM1OjR4/W/fffL8dxYjHUN5D6kt7427FjhyoqKrRmzRodOnRIkyZN0ty5c9Xc3Jzs1AAY1NXVye/368CBA9q9e7fa2tp0yy23qKWlJRazYsUKPfvss3rmmWdUV1enU6dO6Y477khi1gCs6uvrtXXrVk2cOLHDdOob6J8++OADzZo1S4MGDdLzzz+v119/XT//+c81bNiwWMwjjzyixx9/XFu2bNHLL7+soUOHau7cubp8+XISMwfwaR5++GFt3rxZGzZs0BtvvKGHH35YjzzyiNavXx+Lob6B1OdyPtruT4KZM2dq+vTp2rBhgyQpGo2qoKBAy5Yt08qVK5OZGoBuOHPmjHJyclRXV6ebbrpJwWBQ11xzjaqrq3XnnXdKko4dO6axY8dq//79+sIXvpDkjAF8mgsXLmjq1KnatGmTHnjgAU2ePFnr1q2jvoF+bOXKlfrzn/+sl1566YqvO46jvLw8fe9739P3v/99SVIwGFRubq62bdumu+66qzfTBWBw2223KTc3V0899VRs2sKFC5WZmanf/OY31DcwQCT1iL/W1lY1NDSotLQ0Ns3tdqu0tFT79+9PYmYAuisYDEqSsrOzJUkNDQ1qa2vrUO9jxoxRYWEh9Q70E36/X7feemuHOpaob6A/+8Mf/qDi4mJ9+ctfVk5OjqZMmaInn3wy9vqJEyfU1NTUob59Pp9mzpxJfQN93A033KDa2lodP35ckvTaa69p3759mjdvniTqGxgo0pM5+NmzZxWJRJSbm9them5uro4dO5akrAB0VzQaVXl5uWbNmqXx48dLkpqampSRkaGrr766Q2xubq6ampqSkCUAi+3bt+vQoUOqr6//2GvUN9B/vf3229q8ebMqKir0ox/9SPX19frud7+rjIwMlZWVxWr4Svvr1DfQt61cuVKhUEhjxoxRWlqaIpGIHnzwQS1evFiSqG9ggEhq4w9AavL7/Tp69Kj27duX7FQA9IBAIKDly5dr9+7dGjx4cLLTAdCDotGoiouL9dBDD0mSpkyZoqNHj2rLli0qKytLcnYAuuO3v/2tnn76aVVXV+v666/X4cOHVV5erry8POobGECSeqrviBEjlJaW9rG7/p0+fVojR45MUlYAumPp0qV67rnn9OKLLyo/Pz82feTIkWptbdW5c+c6xFPvQN/X0NCg5uZmTZ06Venp6UpPT1ddXZ0ef/xxpaenKzc3l/oG+qlRo0Zp3LhxHaaNHTtWJ0+elKRYDbO/DvQ/9913n1auXKm77rpLEyZM0Ne+9jWtWLFCVVVVkqhvYKBIauMvIyND06ZNU21tbWxaNBpVbW2tSkpKkpgZACvHcbR06VLV1NToT3/6k4qKijq8Pm3aNA0aNKhDvTc2NurkyZPUO9DHzZkzR0eOHNHhw4djj+LiYi1evDj2f+ob6J9mzZqlxsbGDtOOHz+uz372s5KkoqIijRw5skN9h0Ihvfzyy9Q30MddvHhRbnfHn/xpaWmKRqOSqG9goEj6qb4VFRUqKytTcXGxZsyYoXXr1qmlpUVLlixJdmoADPx+v6qrq7Vz5055vd7YdUF8Pp8yMzPl8/l0zz33qKKiQtnZ2crKytKyZctUUlLCHT+BPs7r9cau1/mhoUOHavjw4bHp1DfQP61YsUI33HCDHnroIX3lK1/RK6+8oieeeEJPPPGEJMnlcqm8vFwPPPCArrvuOhUVFWnVqlXKy8vTggULkps8gE80f/58PfjggyosLNT111+vV199VY899pjuvvtuSdQ3MFAkvfG3aNEinTlzRqtXr1ZTU5MmT56sXbt2fewCowD6ts2bN0uSZs+e3WH6r3/9a33jG9+QJP3iF7+Q2+3WwoULFQ6HNXfuXG3atKmXMwWQCNQ30D9Nnz5dNTU1qqys1M9+9jMVFRVp3bp1sYv/S9IPfvADtbS06Fvf+pbOnTunG2+8Ubt27eKan0Aft379eq1atUr33nuvmpublZeXp29/+9tavXp1LIb6BlKfy3EcJ9lJAAAAAAAAAOhZSb3GHwAAAAAAAIDEoPEHAAAAAAAApCAafwAAAAAAAEAKovEHAAAAAAAApCAafwAAAAAAAEAKovEHAAAAAAAApKA+0/gLh8P6yU9+onA4nOxUAPQw6htIXdQ3kLqobyB1Ud/AwOFyHMdJdhKSFAqF5PP5FAwGlZWVlex0APQg6htIXdQ3kLqobyB1Ud/AwNFnjvgDAAAAAAAA0HNo/AEAAAAAAAApKL23B4xGozp16pS8Xq9cLldseigU6vAvgNRBfQOpi/oGUhf1DaQu6hvo/xzH0fnz55WXlye3u/Pj+nr9Gn/vvPOOCgoKenNIAAAAAAAAIOUEAgHl5+d3+nqvH/Hn9XolSTfq35SuQb09PAAAAAAAANCvtatN+/THWJ+tM3E1/jZu3KhHH31UTU1NmjRpktavX68ZM2Z0ad4PT+9N1yClu2j8AQAAAAAAACZ/P3/3o5fRuxLzzT127NihiooKrVmzRocOHdKkSZM0d+5cNTc3x5UnAAAAAAAAgJ5nbvw99thj+uY3v6klS5Zo3Lhx2rJli4YMGaJf/epXicgPAAAAAAAAQBxMjb/W1lY1NDSotLT0H3/A7VZpaan2799/xXnC4bBCoVCHBwAAAAAAAIDEMjX+zp49q0gkotzc3A7Tc3Nz1dTUdMV5qqqq5PP5Yg/u6AsAAAAAAAAknvlUX6vKykoFg8HYIxAIJHpIAAAAAAAAYMAz3dV3xIgRSktL0+nTpztMP336tEaOHHnFeTwejzweT/wZAgAAAAAAADAzHfGXkZGhadOmqba2NjYtGo2qtrZWJSUlPZ4cAAAAAAAAgPiYjviTpIqKCpWVlam4uFgzZszQunXr1NLSoiVLliQiPwAAAAAAAABxMDf+Fi1apDNnzmj16tVqamrS5MmTtWvXro/d8AMAAAAAAABA8rgcx3F6c8BQKCSfz6fZul3prkG9OTQAAAAAAADQ77U7bdqjnQoGg8rKyuo0LuF39QUAAAAAAADQ+2j8AQAAAAAAACnI3Pjbu3ev5s+fr7y8PLlcLv3+979PQFoAAAAAAAAAusPc+GtpadGkSZO0cePGROQDAAAAAAAAoAeY7+o7b948zZs3LxG5AAAAAAAAAOgh5safVTgcVjgcjj0PhUKJHhIAAAAAAAAY8BJ+c4+qqir5fL7Yo6CgINFDAgAAAAAAAANewht/lZWVCgaDsUcgEEj0kAAAAAAAAMCAl/BTfT0ejzweT6KHAQAAAAAAAPARCT/iDwAAAAAAAEDvMx/xd+HCBb311lux5ydOnNDhw4eVnZ2twsLCHk0OAAAAAAAAQHzMjb+DBw/q5ptvjj2vqKiQJJWVlWnbtm09lhgAAAAAAACA+Jkbf7Nnz5bjOInIBQAAAAAAAEAP4Rp/AAAAAAAAQAoyNf6qqqo0ffp0eb1e5eTkaMGCBWpsbExUbgAAAAAAAADiZGr81dXVye/368CBA9q9e7fa2tp0yy23qKWlJVH5AQAAAAAAAIiD6Rp/u3bt6vB827ZtysnJUUNDg2666aYeTQwAAAAAAABA/Mw39/ioYDAoScrOzu40JhwOKxwOx56HQqHuDAkAAAAAAACgC+K+uUc0GlV5eblmzZql8ePHdxpXVVUln88XexQUFMQ7JAAAAAAAAIAuirvx5/f7dfToUW3fvv0T4yorKxUMBmOPQCAQ75AAAAAAAAAAuiiuU32XLl2q5557Tnv37lV+fv4nxno8Hnk8nriSAwAAAAAAABAfU+PPcRwtW7ZMNTU12rNnj4qKihKVFwAAAAAAAIBuMDX+/H6/qqurtXPnTnm9XjU1NUmSfD6fMjMzE5IgAAAAAAAAADvTNf42b96sYDCo2bNna9SoUbHHjh07EpUfAAAAAAAAgDiYT/UFAAAAAAAA0PfFfVdfAAAAAAAAAH0XjT8AAAAAAAAgBZmv8Tdx4kRlZWUpKytLJSUlev755xOVGwAAAAAAAIA4mRp/+fn5Wrt2rRoaGnTw4EF98Ytf1O23366//OUvicoPAAAAAAAAQBxcTjfv2JGdna1HH31U99xzT5fiQ6GQfD6fZut2pbsGdWdoAAAAAAAAYMBpd9q0RzsVDAaVlZXVaZzprr4fFYlE9Mwzz6ilpUUlJSWdxoXDYYXD4djzUCgU75AAAAAAAAAAush8c48jR47oqquuksfj0Xe+8x3V1NRo3LhxncZXVVXJ5/PFHgUFBd1KGAAAAAAAAMCnM5/q29raqpMnTyoYDOp3v/udfvnLX6qurq7T5t+VjvgrKCjgVF8AAAAAAAAgDl091bfb1/grLS3V6NGjtXXr1i7Fc40/AAAAAAAAIH5dbfyZT/X9/6LRaIcj+gAAAAAAAAAkn+nmHpWVlZo3b54KCwt1/vx5VVdXa8+ePXrhhRcSlR8AAAAAAACAOJgaf83Nzfr617+u9957Tz6fTxMnTtQLL7ygL33pS/aRp42T0gd3KfT43ZnmP3/1f9tuWHzNaxfNY6S9etwUH5nyOVO8a/8RU7wkNddcZ4rPvT+OGzsfPmYKb5k/zRSf1ho1xUvSkAN/NcVHRn/GPIarzZjXkUZTuNvrtf19Sc5ncm1jXLxsHuON5SNN8QW7bZ9T4M6IKV6SrtvUaop3v/E38xhv/Xi8KT7rbdvfd9vegiTpmj/alnP54lim3mu2xX+u0BR/Zlrnh6B3JuOC7YoUZ2+zL+cjnu3a9uhDF3NtB8ynhe1X1cg5EDLFH19uew+SNOa+k6b40E3/ZIrPCLab4iXJs9+2jYm2tJjHaJ9j2y6l1zaY4qM3TjbFS1L7UNv2+H/+w75MfX7dJVO8+33bMtgycZQpXpLOTLJd9mVIk/19jzj4gSn+33fsNcXXTLGtByUpUD7VFD/iSJt5jCGNZ03xkb/+zRQf+M8SU7wkfeYl2zKY/r4tXpJcl21nIv1xb40pfnLVvaZ4Sco8a9s3Grb7TfMYrsG2bcD//ovthouZ79vX54PfPW+bIWr/DeCcCNhmcNu2364i+40pXSHbdunS523785L0/gSPeR6LrL/Z98+9L9q23+f+dax5jGiaLf58ofH7juMiaC7jYjvkPfsg7cZ2yIhFtrpwfjzcNoCk89fakhr2X8bfMZJO3/HPpvjs122/AVxR+3fhPviGLd5jr9Xopa6/D5fjkrqwi2Day3zqqacs4QAAAAAAAACSpNvX+AMAAAAAAADQ93Sr8bd27Vq5XC6Vl5f3UDoAAAAAAAAAekLcjb/6+npt3bpVEydO7Ml8AAAAAAAAAPSAuBp/Fy5c0OLFi/Xkk09q2LBhPZ0TAAAAAAAAgG6Kq/Hn9/t16623qrS09FNjw+GwQqFQhwcAAAAAAACAxDLd1VeStm/frkOHDqm+vr5L8VVVVfrpT39qTgwAAAAAAABA/ExH/AUCAS1fvlxPP/20Bg8e3KV5KisrFQwGY49AIBBXogAAAAAAAAC6znTEX0NDg5qbmzV16tTYtEgkor1792rDhg0Kh8NKS0vrMI/H45HH4+mZbAEAAAAAAAB0ianxN2fOHB05cqTDtCVLlmjMmDH64Q9/+LGmHwAAAAAAAIDkMDX+vF6vxo8f32Ha0KFDNXz48I9NBwAAAAAAAJA85pt7dJfjOJKk9ki4y/NEL7nM40RabW+tvf2yeQzHaTXFR4xjuJw2U7wkRS52/XOVpPZIu3kMGfNqb7O9b6ctaoqXpPZoYr8LSXJFHNsMxs/JbVyeJMkx1JEkuaO2eEmKXrJ9Vu3G7y96KWKKl6T2dttnFc9nG71se98R4xBxpGRezmVcPiT7es26DEZa7bUXabPVXvRiPGMY48OmS+RKrcb1h2zbSUmKXjIPYV6mrOtzd7t9G5NmXAajcWwrzdt84xjROLYx7e22/ZbopcQvU9ZthnX5kKRI2LYNiPRCLV26YFtu2+NYoUfC1m1rHMu5df1sXM6t70GKo/bi2I65jPOEztv2W+J53xHjvpF5ey/JFbX9XooY67U9jvW5dRlU1P4bwLrfIse2/bYuT5Lksq4749hmRML2daFFe1sc++fG7yKebYZjTMu6v+aK42N1GRfbeLZjEeOJle0ttmXQiWcZbLOtc+JZr1l/N1hryRW1fxdu8298ey/Lsm/b/vfYD/tsnXE5nxbRw9555x0VFBT05pAAAAAAAABAygkEAsrPz+/09V5v/EWjUZ06dUper1cu1z+6n6FQSAUFBQoEAsrKyurNlAAkGPUNpC7qG0hd1DeQuqhvoP9zHEfnz59XXl6e3O7Oj3Dt9VN93W73J3Yis7KyWPEAKYr6BlIX9Q2kLuobSF3UN9C/+Xy+T40xXqQIAAAAAAAAQH9A4w8AAAAAAABIQX2m8efxeLRmzRp5PJ5kpwKgh1HfQOqivoHURX0DqYv6BgaOXr+5BwAAAAAAAIDE6zNH/AEAAAAAAADoOTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBRE4w8AAAAAAABIQTT+AAAAAAAAgBT0f0cWKIRlOTFdAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from_=85\n", - "to=90\n", - "plt.matshow((first_10[0].x[from_:to,:100]-old_dataset[0].x[from_:to,:100]).abs())\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [ + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n", + "True\n" + ] + }, { - "data": { - "text/plain": [ - "[['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", - " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", - " ['Dany Brillant', 'people.person.gender', 'Male']]" - ] - }, - "execution_count": 102, - "metadata": {}, - "output_type": "execute_result" + "ename": "RuntimeError", + "evalue": "The size of tensor a (0) must match the size of tensor b (1988) at non-singleton dimension 0", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[91], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(old_dataset, first_10):\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mresults_are_close_enough\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m)\u001b[49m)\n", + "Cell \u001b[0;32mIn[90], line 6\u001b[0m, in \u001b[0;36mresults_are_close_enough\u001b[0;34m(ground_truth, new_method, thresh)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_graphs_are_same\u001b[39m(tensor1, tensor2):\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m nx\u001b[38;5;241m.\u001b[39mweisfeiler_lehman_graph_hash(nx\u001b[38;5;241m.\u001b[39mGraph(tensor1\u001b[38;5;241m.\u001b[39mT)) \u001b[38;5;241m==\u001b[39m nx\u001b[38;5;241m.\u001b[39mweisfeiler_lehman_graph_hash(nx\u001b[38;5;241m.\u001b[39mGraph(tensor2\u001b[38;5;241m.\u001b[39mT))\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_sorted_tensors_are_close\u001b[49m\u001b[43m(\u001b[49m\u001b[43mground_truth\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_method\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m \\\n\u001b[1;32m 7\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m _sorted_tensors_are_close(ground_truth\u001b[38;5;241m.\u001b[39medge_attr, new_method\u001b[38;5;241m.\u001b[39medge_attr) \\\n\u001b[1;32m 8\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m _graphs_are_same(ground_truth\u001b[38;5;241m.\u001b[39medge_index, new_method\u001b[38;5;241m.\u001b[39medge_index)\n", + "Cell \u001b[0;32mIn[90], line 3\u001b[0m, in \u001b[0;36mresults_are_close_enough.._sorted_tensors_are_close\u001b[0;34m(tensor1, tensor2)\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_sorted_tensors_are_close\u001b[39m(tensor1, tensor2):\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mall(\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43misclose\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensor1\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensor2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mfloat()\u001b[38;5;241m.\u001b[39mmean(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m>\u001b[39m thresh)\n", + "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (0) must match the size of tensor b (1988) at non-singleton dimension 0" + ] } ], "source": [ - "dataset.raw_dataset[0]['graph'][85:90]" + "for ds in zip(old_dataset, first_10):\n", + " print(results_are_close_enough(*ds))" ] }, { - "cell_type": "code", - "execution_count": 106, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", - " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", - " ['Dany Brillant', 'people.person.gender', 'Male']]" - ] - }, - "execution_count": 106, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "old_dataset.raw_dataset[0]['graph'][85:90]" + "Test Saving and Loading and eq" ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9088" - ] - }, - "execution_count": 107, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "len(old_dataset.raw_dataset[0]['graph'])" + "big_indexer.save('indexer')" ] }, { "cell_type": "code", - "execution_count": 108, + "execution_count": null, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "9088" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" + "ename": "UnicodeDecodeError", + "evalue": "'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mUnicodeDecodeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[47], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m big_indexer \u001b[38;5;241m==\u001b[39m \u001b[43mLargeGraphIndexer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_disk\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mindexer\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/large_graph_indexer.py:292\u001b[0m, in \u001b[0;36mLargeGraphIndexer.from_disk\u001b[0;34m(cls, path)\u001b[0m\n\u001b[1;32m 290\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m(\u001b[38;5;28mlist\u001b[39m(), \u001b[38;5;28mlist\u001b[39m())\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(path \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/edges\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 292\u001b[0m indexer\u001b[38;5;241m.\u001b[39m_edges \u001b[38;5;241m=\u001b[39m \u001b[43mpkl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 293\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(path \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/nodes\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 294\u001b[0m indexer\u001b[38;5;241m.\u001b[39m_nodes \u001b[38;5;241m=\u001b[39m pkl\u001b[38;5;241m.\u001b[39mload(f)\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/codecs.py:322\u001b[0m, in \u001b[0;36mBufferedIncrementalDecoder.decode\u001b[0;34m(self, input, final)\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m, final\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 320\u001b[0m \u001b[38;5;66;03m# decode input (taking the buffer into account)\u001b[39;00m\n\u001b[1;32m 321\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuffer \u001b[38;5;241m+\u001b[39m \u001b[38;5;28minput\u001b[39m\n\u001b[0;32m--> 322\u001b[0m (result, consumed) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfinal\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 323\u001b[0m \u001b[38;5;66;03m# keep undecoded input until the next call\u001b[39;00m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuffer \u001b[38;5;241m=\u001b[39m data[consumed:]\n", + "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte" + ] } ], "source": [ - "len(dataset.raw_dataset[0]['graph'])" + "assert big_indexer == LargeGraphIndexer.from_disk('indexer')" ] }, { From ce5370b9ebe1946ecf6a71c67dbf0fa384348ab3 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 29 May 2024 20:16:51 -0700 Subject: [PATCH 585/752] finished dataloader 2 --- .../llm_plus_gnn/test_eval_gretriever.ipynb | 350 +++++------------- 1 file changed, 86 insertions(+), 264 deletions(-) diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_gretriever.ipynb index 7f805250a5b7..3c6bff20111a 100644 --- a/examples/llm_plus_gnn/test_eval_gretriever.ipynb +++ b/examples/llm_plus_gnn/test_eval_gretriever.ipynb @@ -2,9 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 48, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from typing import List\n", "from torch_geometric.data import InMemoryDataset\n", @@ -17,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -26,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -44,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -56,7 +65,7 @@ "})" ] }, - "execution_count": 51, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -67,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -76,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -93,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -111,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -130,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -140,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -150,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -184,174 +193,14 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "100%|██████████| 100/100 [00:02<00:00, 38.64it/s]\n" + "100%|██████████| 100/100 [00:01<00:00, 55.45it/s]\n" ] } ], @@ -365,16 +214,16 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.5566630363464355" + "0.5066280364990234" ] }, - "execution_count": 63, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -394,7 +243,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -403,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -412,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -424,14 +273,14 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "423763it [00:02, 198983.48it/s]\n" + "423763it [00:02, 204343.22it/s]\n" ] } ], @@ -441,7 +290,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -458,7 +307,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -467,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -476,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -485,7 +334,7 @@ "105413" ] }, - "execution_count": 71, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -496,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -505,7 +354,7 @@ "3095" ] }, - "execution_count": 72, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -516,7 +365,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -525,7 +374,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -583,7 +432,7 @@ ")" ] }, - "execution_count": 74, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -596,7 +445,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -613,7 +462,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -622,7 +471,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -631,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -640,14 +489,14 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "3295it [01:08, 47.83it/s] \n" + "3295it [01:07, 48.80it/s] \n" ] } ], @@ -662,7 +511,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -671,7 +520,7 @@ "torch.Size([105413, 1024])" ] }, - "execution_count": 80, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -682,7 +531,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -691,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -703,14 +552,14 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "97it [00:01, 49.56it/s] \n" + "97it [00:01, 51.27it/s] \n" ] } ], @@ -724,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -733,7 +582,7 @@ "torch.Size([3095, 1024])" ] }, - "execution_count": 84, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -744,7 +593,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -753,7 +602,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -762,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -771,7 +620,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -780,14 +629,14 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 42, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [00:01<00:00, 6.31it/s]\n" + "100%|██████████| 10/10 [00:01<00:00, 5.64it/s]\n" ] } ], @@ -798,7 +647,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -820,21 +669,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 30%|███ | 3/10 [00:10<00:21, 3.03s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SBERT text embedding failed, returning torch.zeros((0, 1024))...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 10/10 [00:35<00:00, 3.55s/it]\n", + "100%|██████████| 10/10 [01:01<00:00, 6.11s/it]\n", "Done!\n" ] } @@ -846,7 +681,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -862,30 +697,24 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "True\n", + "True\n", + "True\n", + "True\n", + "True\n", + "True\n", + "True\n", "True\n", "True\n", "True\n" ] - }, - { - "ename": "RuntimeError", - "evalue": "The size of tensor a (0) must match the size of tensor b (1988) at non-singleton dimension 0", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[91], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(old_dataset, first_10):\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mresults_are_close_enough\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m)\u001b[49m)\n", - "Cell \u001b[0;32mIn[90], line 6\u001b[0m, in \u001b[0;36mresults_are_close_enough\u001b[0;34m(ground_truth, new_method, thresh)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_graphs_are_same\u001b[39m(tensor1, tensor2):\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m nx\u001b[38;5;241m.\u001b[39mweisfeiler_lehman_graph_hash(nx\u001b[38;5;241m.\u001b[39mGraph(tensor1\u001b[38;5;241m.\u001b[39mT)) \u001b[38;5;241m==\u001b[39m nx\u001b[38;5;241m.\u001b[39mweisfeiler_lehman_graph_hash(nx\u001b[38;5;241m.\u001b[39mGraph(tensor2\u001b[38;5;241m.\u001b[39mT))\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_sorted_tensors_are_close\u001b[49m\u001b[43m(\u001b[49m\u001b[43mground_truth\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_method\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m \\\n\u001b[1;32m 7\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m _sorted_tensors_are_close(ground_truth\u001b[38;5;241m.\u001b[39medge_attr, new_method\u001b[38;5;241m.\u001b[39medge_attr) \\\n\u001b[1;32m 8\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m _graphs_are_same(ground_truth\u001b[38;5;241m.\u001b[39medge_index, new_method\u001b[38;5;241m.\u001b[39medge_index)\n", - "Cell \u001b[0;32mIn[90], line 3\u001b[0m, in \u001b[0;36mresults_are_close_enough.._sorted_tensors_are_close\u001b[0;34m(tensor1, tensor2)\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_sorted_tensors_are_close\u001b[39m(tensor1, tensor2):\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mall(\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43misclose\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensor1\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensor2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mfloat()\u001b[38;5;241m.\u001b[39mmean(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m>\u001b[39m thresh)\n", - "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (0) must match the size of tensor b (1988) at non-singleton dimension 0" - ] } ], "source": [ @@ -902,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -911,27 +740,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, - "outputs": [ - { - "ename": "UnicodeDecodeError", - "evalue": "'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mUnicodeDecodeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[47], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m big_indexer \u001b[38;5;241m==\u001b[39m \u001b[43mLargeGraphIndexer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_disk\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mindexer\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/large_graph_indexer.py:292\u001b[0m, in \u001b[0;36mLargeGraphIndexer.from_disk\u001b[0;34m(cls, path)\u001b[0m\n\u001b[1;32m 290\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m(\u001b[38;5;28mlist\u001b[39m(), \u001b[38;5;28mlist\u001b[39m())\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(path \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/edges\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 292\u001b[0m indexer\u001b[38;5;241m.\u001b[39m_edges \u001b[38;5;241m=\u001b[39m \u001b[43mpkl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 293\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(path \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/nodes\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 294\u001b[0m indexer\u001b[38;5;241m.\u001b[39m_nodes \u001b[38;5;241m=\u001b[39m pkl\u001b[38;5;241m.\u001b[39mload(f)\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/codecs.py:322\u001b[0m, in \u001b[0;36mBufferedIncrementalDecoder.decode\u001b[0;34m(self, input, final)\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m, final\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 320\u001b[0m \u001b[38;5;66;03m# decode input (taking the buffer into account)\u001b[39;00m\n\u001b[1;32m 321\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuffer \u001b[38;5;241m+\u001b[39m \u001b[38;5;28minput\u001b[39m\n\u001b[0;32m--> 322\u001b[0m (result, consumed) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfinal\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 323\u001b[0m \u001b[38;5;66;03m# keep undecoded input until the next call\u001b[39;00m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuffer \u001b[38;5;241m=\u001b[39m data[consumed:]\n", - "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte" - ] - } - ], + "outputs": [], "source": [ "assert big_indexer == LargeGraphIndexer.from_disk('indexer')" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, From dc3f9f43fce509a656c49a22de55d9bf835b3cee Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 30 May 2024 14:19:04 -0700 Subject: [PATCH 586/752] add to data transform --- examples/llm_plus_gnn/large_graph_indexer.py | 115 ++++++++++++++---- .../llm_plus_gnn/test_eval_gretriever.ipynb | 23 ++-- 2 files changed, 108 insertions(+), 30 deletions(-) diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/examples/llm_plus_gnn/large_graph_indexer.py index b55d551ed707..ecaca03703f6 100644 --- a/examples/llm_plus_gnn/large_graph_indexer.py +++ b/examples/llm_plus_gnn/large_graph_indexer.py @@ -14,10 +14,12 @@ Sequence, Set, Tuple, + Type, ) import torch -from torch_geometric.data import Data +from torch_geometric.data import Data, FeatureStore, GraphStore +from torch_geometric.distributed import LocalFeatureStore, LocalGraphStore # Is there a multiprocessing-friendly implementation of this? @@ -28,12 +30,19 @@ def ordered_set(values: Iterable[Hashable]) -> List[Hashable]: return list(dict.fromkeys(values)) +# TODO: Refactor Node and Edge funcs and attrs to be accessible via an Enum? + NODE_PID = "pid" +NODE_KEYS = {NODE_PID} + EDGE_PID = "e_pid" EDGE_HEAD = "h" EDGE_RELATION = "r" EDGE_TAIL = "t" +EDGE_INDEX = "edge_idx" + +EDGE_KEYS = {EDGE_PID, EDGE_HEAD, EDGE_RELATION, EDGE_TAIL, EDGE_INDEX} @dataclass @@ -77,6 +86,12 @@ def __init__( if node_attr is not None: # TODO: Validity checks btw nodes and node_attr self.node_attr = node_attr + if NODE_KEYS & set(self.node_attr.keys()) != NODE_KEYS: + raise AttributeError( + f"Invalid node_attr object. Missing {NODE_KEYS - set(self.node_attr.keys())}" + ) + elif self.node_attr[NODE_PID] != nodes: + raise AttributeError(f"Nodes provided do not match those in node_attr") else: self.node_attr = dict() self.node_attr[NODE_PID] = nodes @@ -87,20 +102,29 @@ def __init__( if edge_attr is not None: # TODO: Validity checks btw edges and edge_attr self.edge_attr = edge_attr - for i, tup in enumerate(edges): - self._edges[tup] = i + + if EDGE_KEYS & set(self.edge_attr.keys()) != EDGE_KEYS: + raise AttributeError( + f"Invalid edge_attr object. Missing {EDGE_KEYS - set(self.edge_attr.keys())}" + ) + elif self.node_attr[EDGE_PID] != edges: + raise AttributeError(f"Edges provided do not match those in edge_attr") + else: self.edge_attr = dict() - for default_key in [EDGE_HEAD, EDGE_RELATION, EDGE_TAIL, EDGE_PID]: + for default_key in EDGE_KEYS: self.edge_attr[default_key] = list() + self.edge_attr[EDGE_PID] = edges for i, tup in enumerate(edges): h, r, t = tup self.edge_attr[EDGE_HEAD].append(h) self.edge_attr[EDGE_RELATION].append(r) self.edge_attr[EDGE_TAIL].append(t) - self.edge_attr[EDGE_PID].append(tup) - self._edges[tup] = i + self.edge_attr[EDGE_INDEX].append((self._nodes[h], self._nodes[t])) + + for i, tup in enumerate(edges): + self._edges[tup] = i @classmethod def from_triplets( @@ -174,10 +198,22 @@ def add_node_feature( def get_node_features( self, feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None ) -> List[Any]: + if feature_name in self._mapped_node_features: + values = self.node_attr[feature_name].values + else: + values = self.node_attr[feature_name] + if isinstance(values, torch.Tensor): + idxs = list( + self.get_node_features_iter(feature_name, pids, index_only=True) + ) + return values[idxs] return list(self.get_node_features_iter(feature_name, pids)) def get_node_features_iter( - self, feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None + self, + feature_name: str = NODE_PID, + pids: Optional[Iterable[Hashable]] = None, + index_only: bool = False, ) -> Iterator[Any]: if pids is None: pids = self.node_attr[NODE_PID] @@ -195,11 +231,17 @@ def get_node_features_iter( idx = self._nodes[pid] from_feature_val = self.node_attr[from_feature_name][idx] to_feature_idx = feature_mapping[from_feature_val] - yield to_feature_vals[to_feature_idx] + if index_only: + yield to_feature_idx + else: + yield to_feature_vals[to_feature_idx] else: for pid in pids: idx = self._nodes[pid] - yield self.node_attr[feature_name][idx] + if index_only: + yield idx + else: + yield self.node_attr[feature_name][idx] def get_unique_edge_features(self, feature_name: str = EDGE_PID) -> List[Hashable]: try: @@ -239,13 +281,23 @@ def get_edge_features( self, feature_name: str = EDGE_PID, pids: Optional[Iterable[Hashable]] = None, - ) -> List[Any]: + ) -> Sequence[Any]: + if feature_name in self._mapped_edge_features: + values = self.edge_attr[feature_name].values + else: + values = self.edge_attr[feature_name] + if isinstance(values, torch.Tensor): + idxs = list( + self.get_edge_features_iter(feature_name, pids, index_only=True) + ) + return values[idxs] return list(self.get_edge_features_iter(feature_name, pids)) def get_edge_features_iter( self, feature_name: str = EDGE_PID, pids: Optional[Iterable[TripletLike]] = None, + index_only: bool = False, ) -> Iterator[Any]: if pids is None: pids = self.edge_attr[EDGE_PID] @@ -263,11 +315,17 @@ def get_edge_features_iter( idx = self._edges[pid] from_feature_val = self.edge_attr[from_feature_name][idx] to_feature_idx = feature_mapping[from_feature_val] - yield to_feature_vals[to_feature_idx] + if index_only: + yield to_feature_idx + else: + yield to_feature_vals[to_feature_idx] else: for pid in pids: idx = self._edges[pid] - yield self.edge_attr[feature_name][idx] + if index_only: + yield idx + else: + yield self.edge_attr[feature_name][idx] def to_triplets(self) -> Iterator[TripletLike]: return iter(self.edge_attr[EDGE_PID]) @@ -344,6 +402,26 @@ def __eq__(self, value: "LargeGraphIndexer") -> bool: eq &= self.edge_attr[k] == value.edge_attr[k] return eq + def _to_data_attrs( + self, node_feature_name: str, edge_feature_name: Optional[str] = None + ) -> Tuple[torch.Tensor, torch.LongTensor, Optional[torch.Tensor]]: + x = torch.Tensor(self.get_node_features(node_feature_name)) + edge_index = torch.t(torch.LongTensor(self.get_edge_features(EDGE_INDEX))) + edge_attr = ( + self.get_edge_features(edge_feature_name) + if edge_feature_name is not None + else None + ) + return x, edge_index, edge_attr + + def to_data( + self, node_feature_name: str, edge_feature_name: Optional[str] = None + ) -> Data: + x, edge_index, edge_attr = self._to_data_attrs( + node_feature_name, edge_feature_name + ) + return Data(x=x, edge_index=edge_index, edge_attr=edge_attr) + def get_features_for_triplets( indexer: LargeGraphIndexer, @@ -375,17 +453,10 @@ def apply_transform(trips): feature_name=edge_feature_name, pids=edge_keys ) - edge_index = [] - for h, t in zip( - small_graph_indexer.get_edge_features_iter("h", triplets), - small_graph_indexer.get_edge_features_iter("t", triplets), - ): - edge_index.append( - [small_graph_indexer._nodes[h], small_graph_indexer._nodes[t]] - ) + edge_index = small_graph_indexer.get_edge_features(EDGE_INDEX, triplets) - x = torch.stack(node_feats, 0) - edge_attr = torch.stack(edge_feats, 0) + x = torch.Tensor(node_feats) + edge_attr = torch.Tensor(edge_feats) edge_index = torch.t(torch.LongTensor(edge_index)) return Data( x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(node_feats) diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_gretriever.ipynb index 3c6bff20111a..afcd7d52c0c3 100644 --- a/examples/llm_plus_gnn/test_eval_gretriever.ipynb +++ b/examples/llm_plus_gnn/test_eval_gretriever.ipynb @@ -200,7 +200,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 100/100 [00:01<00:00, 55.45it/s]\n" + "100%|██████████| 100/100 [00:01<00:00, 54.26it/s]\n" ] } ], @@ -220,7 +220,7 @@ { "data": { "text/plain": [ - "0.5066280364990234" + "0.6767024993896484" ] }, "execution_count": 16, @@ -280,7 +280,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "423763it [00:02, 204343.22it/s]\n" + "0it [00:00, ?it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "423763it [00:02, 194623.85it/s]\n" ] } ], @@ -290,7 +297,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -496,7 +503,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "3295it [01:07, 48.80it/s] \n" + "3295it [01:08, 48.15it/s] \n" ] } ], @@ -559,7 +566,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "97it [00:01, 51.27it/s] \n" + "97it [00:01, 50.54it/s] \n" ] } ], @@ -636,7 +643,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [00:01<00:00, 5.64it/s]\n" + "100%|██████████| 10/10 [00:01<00:00, 6.13it/s]\n" ] } ], @@ -669,7 +676,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [01:01<00:00, 6.11s/it]\n", + "100%|██████████| 10/10 [01:01<00:00, 6.12s/it]\n", "Done!\n" ] } From 4488d708c1256c17e3ba62f3f4b1b2f470019a14 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 30 May 2024 17:08:38 -0700 Subject: [PATCH 587/752] init commit of dataset --- examples/llm_plus_gnn/updated_qsp_dataset.py | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 examples/llm_plus_gnn/updated_qsp_dataset.py diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py new file mode 100644 index 000000000000..e5efb262b007 --- /dev/null +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -0,0 +1,132 @@ +from itertools import chain +from typing import Iterator, Optional + +from large_graph_indexer import ( + EDGE_RELATION, + LargeGraphIndexer, + TripletLike, + get_features_for_triplets, +) +from torch_geometric.datasets.web_qsp_dataset import * + + +def preprocess_triplet(triplet: TripletLike) -> TripletLike: + return tuple([s.lower() for s in triplet]) + + +class UpdatedWebQSPDataset(WebQSPDataset): + + def __init__( + self, + root: str = "", + force_reload: bool = False, + whole_graph_retrieval: bool = False, + limit: int = -1, + ) -> None: + self.limit = limit + self.whole_graph_retrieval = whole_graph_retrieval + super().__init__(root, force_reload) + + @property + def raw_file_names(self) -> List[str]: + return ["raw_data", "split_idxs"] + + @property + def processed_file_names(self) -> List[str]: + return [ + "list_of_graphs.pt", + "pre_filter.pt", + "pre_transform.pt", + "large_graph.pt", + ] + + def _save_raw_data(self) -> None: + self.raw_dataset.save_to_disk(self.raw_paths[0]) + torch.save(self.split_idxs, self.raw_paths[1]) + + def _load_raw_data(self) -> None: + self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) + self.split_idxs = torch.load(self.raw_paths[1]) + + def download(self) -> None: + super().download() + if self.limit >= 0: + self.raw_dataset = self.raw_dataset.select(range(self.limit)) + self._save_raw_data() + + def _get_trips(self) -> Iterator[TripletLike]: + return chain.from_iterable((iter(ds["graph"]) for ds in self.raw_dataset)) + + def _build_graph(self) -> None: + trips = self._get_trips() + self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( + trips, pre_transform=preprocess_triplet + ) + + # Nodes: + nodes = self.indexer.get_unique_node_features() + x = text2embedding(self.model, self.device, nodes) + self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) + + # Edges: + edges = self.indexer.get_unique_edge_features(feature_name=EDGE_RELATION) + edge_attr = text2embedding(self.model, self.device, edges) + self.indexer.add_edge_feature( + new_feature_name="edge_attr", + new_feature_vals=edge_attr, + map_from_feature=EDGE_RELATION, + ) + + self.graph = self.indexer.to_data( + node_feature_name="x", edge_feature_name="edge_attr" + ) + self.save(self.graph, self.processed_paths[3]) + + def _retrieve_subgraphs(self) -> None: + print("Encoding questions...") + q_embs = text2embedding(self.model, self.device, self.questions) + list_of_graphs = [] + print("Retrieving subgraphs...") + for index in tqdm(range(len(self.raw_dataset))): + data_i = self.raw_dataset[index] + local_trips = data_i["graph"] + if self.whole_graph_retrieval: + graph = self.graph + else: + graph = get_features_for_triplets( + self.indexer, local_trips, pre_transform=preprocess_triplet + ) + pcst_subgraph, desc = retrieval_via_pcst( + graph, + q_embs[index], + self.textual_nodes, + self.textual_edges, + topk=3, + topk_e=5, + cost_e=0.5, + ) + question = f"Question: {data_i['question']}\nAnswer: " + label = ("|").join(data_i["answer"]).lower() + + pcst_subgraph["question"] = question + pcst_subgraph["label"] = label + pcst_subgraph["desc"] - desc + list_of_graphs.append(pcst_subgraph.to("cpu")) + self.save(list_of_graphs, self.processed_paths[0]) + + def process(self) -> None: + pretrained_repo = "sentence-transformers/all-roberta-large-v1" + self.model = SentenceTransformer(pretrained_repo) + self.model.to(self.device) + self.model.eval() + self.questions = [ds["question"] for ds in self.raw_dataset] + print("Encoding graph...") + self._build_graph() + self.textual_nodes = pd.DataFrame( + {"node_attr": self.indexer.get_edge_features()} + ) + self.textual_nodes["node_id"] = self.textual_nodes.index + self.textual_edges = pd.DataFrame( + self.indexer.get_edge_features, columns=["src", "edge_attr", "dst"] + ) + self._retrieve_subgraphs() From eb9b7617684e875dc5700c71c088db0e09b8f856 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 30 May 2024 19:13:42 -0700 Subject: [PATCH 588/752] debug pcst part 1 --- examples/llm_plus_gnn/large_graph_indexer.py | 13 +- examples/llm_plus_gnn/raw_qsp_dataset.py | 48 ++- ...nb => test_eval_large_graph_indexer.ipynb} | 27 +- .../test_eval_updated_webqsp.ipynb | 285 ++++++++++++++++++ examples/llm_plus_gnn/updated_qsp_dataset.py | 56 ++-- torch_geometric/datasets/web_qsp_dataset.py | 59 ++-- 6 files changed, 415 insertions(+), 73 deletions(-) rename examples/llm_plus_gnn/{test_eval_gretriever.ipynb => test_eval_large_graph_indexer.ipynb} (96%) create mode 100644 examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/examples/llm_plus_gnn/large_graph_indexer.py index ecaca03703f6..1a9151d26594 100644 --- a/examples/llm_plus_gnn/large_graph_indexer.py +++ b/examples/llm_plus_gnn/large_graph_indexer.py @@ -426,8 +426,8 @@ def to_data( def get_features_for_triplets( indexer: LargeGraphIndexer, triplets: Iterable[TripletLike], - node_feature_name: str = "embs", - edge_feature_name: str = "embs", + node_feature_name: str = "x", + edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> Data: @@ -458,6 +458,13 @@ def apply_transform(trips): x = torch.Tensor(node_feats) edge_attr = torch.Tensor(edge_feats) edge_index = torch.t(torch.LongTensor(edge_index)) - return Data( + data_obj = Data( x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(node_feats) ) + # needed for mappings + data_obj[NODE_PID] = node_keys + data_obj[EDGE_PID] = edge_keys + data_obj["node_idx"] = [indexer._nodes[k] for k in node_keys] + data_obj["edge_idx"] = [indexer._edges[e] for e in edge_keys] + + return data_obj diff --git a/examples/llm_plus_gnn/raw_qsp_dataset.py b/examples/llm_plus_gnn/raw_qsp_dataset.py index 1bdb214d7019..d8a3cf0a64e0 100644 --- a/examples/llm_plus_gnn/raw_qsp_dataset.py +++ b/examples/llm_plus_gnn/raw_qsp_dataset.py @@ -1,6 +1,7 @@ -from torch_geometric.datasets.web_qsp_dataset import * from typing import Optional +from torch_geometric.datasets.web_qsp_dataset import * + class RawWebQSPDataset(WebQSPDataset): @@ -9,9 +10,11 @@ def __init__( root: str = "", force_reload: bool = False, with_process: bool = False, + with_pcst: bool = False, limit: Optional[int] = None, ) -> None: self.with_process = with_process + self.with_pcst = with_pcst self.limit = limit if self.with_process: super().__init__(root, force_reload) @@ -30,6 +33,13 @@ def raw_file_names(self) -> List[str]: else: return [] + @property + def processed_file_names(self) -> List[str]: + if self.with_process: + return super().processed_file_names + ["raw_graphs.pt"] + else: + return super().processed_file_names + def _save_raw_data(self) -> None: self.raw_dataset.save_to_disk(self.raw_paths[0]) torch.save(self.split_idxs, self.raw_paths[1]) @@ -49,11 +59,13 @@ def process(self) -> None: self.model = SentenceTransformer(pretrained_repo) self.model.to(self.device) self.model.eval() - # self.questions = [i["question"] for i in self.raw_dataset] list_of_graphs = [] - # encode questions - # print("Encoding questions...") - # q_embs = text2embedding(self.model, self.device, self.questions) + self.raw_graphs = [] + if self.with_pcst: + self.questions = [i["question"] for i in self.raw_dataset] + print("Encoding questions...") + q_embs = text2embedding(self.model, self.device, self.questions) + print("Encoding graphs...") limit = self.limit if self.limit else len(self.raw_dataset) for index in tqdm(range(limit)): @@ -92,12 +104,22 @@ def process(self) -> None: edge_attr=edge_attr, num_nodes=len(nodes), ).to("cpu") - list_of_graphs.append(raw_graph) - # psct_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], - # nodes, edges, topk=3, - # topk_e=5, cost_e=0.5) - # psct_subgraph["question"] = question - # psct_subgraph["label"] = label - # psct_subgraph["desc"] = desc - # list_of_graphs.append(psct_subgraph.to("cpu")) + self.raw_graphs.append(raw_graph) + if self.with_pcst: + psct_subgraph, desc = retrieval_via_pcst( + raw_graph, + q_embs[index], + nodes, + edges, + topk=3, + topk_e=5, + cost_e=0.5, + ) + psct_subgraph["question"] = question + psct_subgraph["label"] = label + psct_subgraph["desc"] = desc + list_of_graphs.append(psct_subgraph.to("cpu")) + else: + list_of_graphs.append(raw_graph) + torch.save(self.raw_graphs, self.processed_paths[-1]) self.save(list_of_graphs, self.processed_paths[0]) diff --git a/examples/llm_plus_gnn/test_eval_gretriever.ipynb b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb similarity index 96% rename from examples/llm_plus_gnn/test_eval_gretriever.ipynb rename to examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb index afcd7d52c0c3..80220ddcf7ab 100644 --- a/examples/llm_plus_gnn/test_eval_gretriever.ipynb +++ b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb @@ -200,7 +200,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 100/100 [00:01<00:00, 54.26it/s]\n" + "100%|██████████| 100/100 [00:01<00:00, 51.52it/s]\n" ] } ], @@ -220,7 +220,7 @@ { "data": { "text/plain": [ - "0.6767024993896484" + "0.6365907192230225" ] }, "execution_count": 16, @@ -280,14 +280,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "0it [00:00, ?it/s]" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "423763it [00:02, 194623.85it/s]\n" + "423763it [00:02, 198021.34it/s]\n" ] } ], @@ -297,7 +290,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -503,7 +496,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "3295it [01:08, 48.15it/s] \n" + "3295it [01:07, 48.79it/s] \n" ] } ], @@ -542,7 +535,7 @@ "metadata": {}, "outputs": [], "source": [ - "big_indexer.add_node_feature(new_feature_name=\"embs\", new_feature_vals=node_embs)" + "big_indexer.add_node_feature(new_feature_name=\"x\", new_feature_vals=node_embs)" ] }, { @@ -566,7 +559,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "97it [00:01, 50.54it/s] \n" + "97it [00:01, 51.39it/s] \n" ] } ], @@ -604,7 +597,7 @@ "metadata": {}, "outputs": [], "source": [ - "big_indexer.add_edge_feature(new_feature_name=\"embs\", new_feature_vals=edge_embs, map_from_feature=\"r\")" + "big_indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_embs, map_from_feature=\"r\")" ] }, { @@ -643,7 +636,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [00:01<00:00, 6.13it/s]\n" + "100%|██████████| 10/10 [00:01<00:00, 6.19it/s]\n" ] } ], @@ -676,7 +669,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [01:01<00:00, 6.12s/it]\n", + "100%|██████████| 10/10 [01:00<00:00, 6.07s/it]\n", "Done!\n" ] } diff --git a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb new file mode 100644 index 000000000000..0fc7f525158f --- /dev/null +++ b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb @@ -0,0 +1,285 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from updated_qsp_dataset import UpdatedWebQSPDataset\n", + "from raw_qsp_dataset import RawWebQSPDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inherit model weights from sentence-transformers/all-roberta-large-v1\n", + "Encoding questions...\n", + "Encoding graphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 2/2 [00:08<00:00, 4.30s/it]\n", + "Done!\n", + "Processing...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inherit model weights from sentence-transformers/all-roberta-large-v1\n", + "Encoding graph...\n", + "Encoding questions...\n", + "Retrieving subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 2/2 [00:00<00:00, 3.56it/s]\n", + "Done!\n" + ] + } + ], + "source": [ + "old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", + "new_dataset = UpdatedWebQSPDataset(root='updated_dataset',force_reload=True, limit=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Data(x=[15, 1024], edge_index=[2, 20], edge_attr=[20, 1024], question='Question: what is the name of justin bieber brother\n", + "Answer: ', label='jaxon bieber', desc='node_id,node_attr\n", + "249,m.0129jzth\n", + "434,jeremy bieber\n", + "880,m.0gxnp5d\n", + "892,m.0gxnnwc\n", + "1178,justin bieber\n", + "1239,yves bole\n", + "1563,english language\n", + "1569,jazmyn bieber\n", + "1742,pattie mallette\n", + "1858,m.0gxnp5m\n", + "1888,this is justin bieber\n", + "2050,jaxon bieber\n", + "2205,m.0gxnnwp\n", + "2436,m.0wfn4pm\n", + "2620,m.0gxnp5x\n", + "\n", + "src,edge_attr,dst\n", + "1239,people.person.languages,1563\n", + "1888,film.film.language,1563\n", + "1239,influence.influence_node.influenced_by,1178\n", + "1742,people.person.children,1178\n", + "2050,people.person.parents,434\n", + "2436,people.sibling_relationship.sibling,1742\n", + "2050,people.person.sibling_s,2205\n", + "1178,base.popstra.celebrity.hangout,2620\n", + "892,people.sibling_relationship.sibling,1569\n", + "249,people.sibling_relationship.sibling,1239\n", + "1742,people.person.sibling_s,2436\n", + "2205,people.sibling_relationship.sibling,2050\n", + "1239,people.person.sibling_s,249\n", + "1178,people.person.sibling_s,892\n", + "1178,base.popstra.celebrity.hangout,880\n", + "1569,people.person.sibling_s,892\n", + "892,people.sibling_relationship.sibling,1178\n", + "1178,people.person.sibling_s,2205\n", + "1178,base.popstra.celebrity.hangout,1858\n", + "2205,people.sibling_relationship.sibling,1178\n", + "', num_nodes=15)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "new_dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Data(x=[12, 1024], edge_index=[2, 13], edge_attr=[13, 1024], question='Question: what is the name of justin bieber brother\n", + "Answer: ', label='jaxon bieber', desc='node_id,node_attr\n", + "15,justin bieber\n", + "151,pattie mallette\n", + "286,english language\n", + "294,jaxon bieber\n", + "346,yves bole\n", + "356,jeremy bieber\n", + "452,jazmyn bieber\n", + "545,m.0wfn4pm\n", + "551,m.0gxnnwp\n", + "933,m.0gxnnwc\n", + "1032,this is justin bieber\n", + "1359,m.0129jzth\n", + "\n", + "src,edge_attr,dst\n", + "346,people.person.languages,286\n", + "1032,film.film.language,286\n", + "346,influence.influence_node.influenced_by,15\n", + "151,people.person.children,15\n", + "294,people.person.parents,356\n", + "294,people.person.sibling_s,551\n", + "452,people.person.sibling_s,933\n", + "545,people.sibling_relationship.sibling,151\n", + "933,people.sibling_relationship.sibling,452\n", + "1359,people.sibling_relationship.sibling,346\n", + "551,people.sibling_relationship.sibling,294\n", + "933,people.sibling_relationship.sibling,15\n", + "551,people.sibling_relationship.sibling,15\n", + "', num_nodes=12)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "old_dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_dataset.raw_graphs[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "old_dataset.raw_graphs[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import networkx as nx\n", + "from torch_geometric.data import Data\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", + " def _sorted_tensors_are_close(tensor1, tensor2):\n", + " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", + " def _graphs_are_same(tensor1, tensor2):\n", + " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", + " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", + " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", + " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results_are_close_enough(old_dataset.raw_graphs[0], new_dataset.raw_graphs[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "old_dataset.questions[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_dataset.questions[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py index e5efb262b007..e1b2056698ba 100644 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -1,3 +1,4 @@ +import os from itertools import chain from typing import Iterator, Optional @@ -37,7 +38,8 @@ def processed_file_names(self) -> List[str]: "list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt", - "large_graph.pt", + "raw_graphs.pt", + "large_graph_indexer", ] def _save_raw_data(self) -> None: @@ -45,8 +47,10 @@ def _save_raw_data(self) -> None: torch.save(self.split_idxs, self.raw_paths[1]) def _load_raw_data(self) -> None: - self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) - self.split_idxs = torch.load(self.raw_paths[1]) + if not hasattr(self, "raw_dataset"): + self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) + if not hasattr(self, "split_idxs"): + self.split_idxs = torch.load(self.raw_paths[1]) def download(self) -> None: super().download() @@ -77,30 +81,36 @@ def _build_graph(self) -> None: map_from_feature=EDGE_RELATION, ) - self.graph = self.indexer.to_data( - node_feature_name="x", edge_feature_name="edge_attr" - ) - self.save(self.graph, self.processed_paths[3]) + self.indexer.save(self.processed_paths[-1]) def _retrieve_subgraphs(self) -> None: print("Encoding questions...") + self.questions = [ds["question"] for ds in self.raw_dataset] q_embs = text2embedding(self.model, self.device, self.questions) list_of_graphs = [] + self.raw_graphs = [] print("Retrieving subgraphs...") + textual_nodes = self.textual_nodes + textual_edges = self.textual_edges for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] local_trips = data_i["graph"] if self.whole_graph_retrieval: - graph = self.graph + graph = self.indexer.to_data( + node_feature_name="x", edge_feature_name="edge_attr" + ) else: graph = get_features_for_triplets( self.indexer, local_trips, pre_transform=preprocess_triplet ) + textual_nodes = self.textual_nodes.iloc[graph["node_idx"]] + textual_edges = self.textual_edges.iloc[graph["edge_idx"]] + self.raw_graphs.append(graph) pcst_subgraph, desc = retrieval_via_pcst( graph, q_embs[index], - self.textual_nodes, - self.textual_edges, + textual_nodes, + textual_edges, topk=3, topk_e=5, cost_e=0.5, @@ -110,23 +120,35 @@ def _retrieve_subgraphs(self) -> None: pcst_subgraph["question"] = question pcst_subgraph["label"] = label - pcst_subgraph["desc"] - desc + pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) + torch.save(self.raw_graphs, self.processed_paths[-2]) self.save(list_of_graphs, self.processed_paths[0]) def process(self) -> None: + self._load_raw_data() pretrained_repo = "sentence-transformers/all-roberta-large-v1" self.model = SentenceTransformer(pretrained_repo) self.model.to(self.device) self.model.eval() - self.questions = [ds["question"] for ds in self.raw_dataset] - print("Encoding graph...") - self._build_graph() - self.textual_nodes = pd.DataFrame( - {"node_attr": self.indexer.get_edge_features()} + if not os.path.exists(self.processed_dir[-1]): + print("Encoding graph...") + self._build_graph() + else: + print("Loading graph...") + self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) + self.textual_nodes = pd.DataFrame.from_dict( + {"node_attr": self.indexer.get_node_features()} ) self.textual_nodes["node_id"] = self.textual_nodes.index + self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] self.textual_edges = pd.DataFrame( - self.indexer.get_edge_features, columns=["src", "edge_attr", "dst"] + self.indexer.get_edge_features(), columns=["src", "edge_attr", "dst"] ) + self.textual_edges["src"] = [ + self.indexer._nodes[h] for h in self.textual_edges["src"] + ] + self.textual_edges["dst"] = [ + self.indexer._nodes[h] for h in self.textual_edges["dst"] + ] self._retrieve_subgraphs() diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 998344c3f53f..81b6b782f348 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -18,12 +18,14 @@ try: from pcst_fast import pcst_fast + WITH_PCST = True except ImportError: WITH_PCST = False try: import datasets + WITH_DATASETS = True except ImportError: WITH_DATASETS = False @@ -41,10 +43,17 @@ def retrieval_via_pcst( ) -> Tuple[Data, str]: c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - graph = Data(x=graph.x, edge_index=graph.edge_index, - edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + desc = ( + textual_nodes.to_csv(index=False) + + "\n" + + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) + ) + graph = Data( + x=graph.x, + edge_index=graph.edge_index, + edge_attr=graph.edge_attr, + num_nodes=graph.num_nodes, + ) return graph, desc root = -1 # unrooted @@ -106,8 +115,9 @@ def retrieval_via_pcst( costs = np.array(costs + virtual_costs) edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, - pruning, verbosity_level) + vertices, edges = pcst_fast( + edges, prizes, costs, root, num_clusters, pruning, verbosity_level + ) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] @@ -119,13 +129,16 @@ def retrieval_via_pcst( edge_index = graph.edge_index[:, selected_edges] selected_nodes = np.unique( - np.concatenate( - [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + np.concatenate([selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()]) + ) n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = n.to_csv(index=False) + "\n" + e.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) + desc = ( + n.to_csv(index=False) + + "\n" + + e.to_csv(index=False, columns=["src", "edge_attr", "dst"]) + ) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -134,8 +147,9 @@ def retrieval_via_pcst( src = [mapping[i] for i in edge_index[0].tolist()] dst = [mapping[i] for i in edge_index[1].tolist()] edge_index = torch.LongTensor([src, dst]) - data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(selected_nodes)) + data = Data( + x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes) + ) return data, desc @@ -156,17 +170,17 @@ class WebQSPDataset(InMemoryDataset): force_reload (bool, optional): Whether to re-process the dataset. (default: :obj:`False`) """ + def __init__( self, root: str = "", force_reload: bool = False, ) -> None: self._check_dependencies() - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) - + def _check_dependencies(self) -> None: missing_imports = False missing_str_list = [] @@ -192,15 +206,14 @@ def processed_file_names(self) -> List[str]: def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( - [dataset["train"], dataset["validation"], dataset["test"]]) + [dataset["train"], dataset["validation"], dataset["test"]] + ) self.split_idxs = { - "train": - torch.arange(len(dataset["train"])), - "val": - torch.arange(len(dataset["validation"])) + len(dataset["train"]), - "test": - torch.arange(len(dataset["test"])) + len(dataset["train"]) + - len(dataset["validation"]) + "train": torch.arange(len(dataset["train"])), + "val": torch.arange(len(dataset["validation"])) + len(dataset["train"]), + "test": torch.arange(len(dataset["test"])) + + len(dataset["train"]) + + len(dataset["validation"]), } def process(self) -> None: From 85da0e725613b17b87b6f69fb8c1f2938116d9ed Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 31 May 2024 15:26:56 -0700 Subject: [PATCH 589/752] push what i have for now --- examples/llm_plus_gnn/updated_qsp_dataset.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py index e1b2056698ba..e5e5de84338d 100644 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -12,7 +12,8 @@ def preprocess_triplet(triplet: TripletLike) -> TripletLike: - return tuple([s.lower() for s in triplet]) + h, r, t = triplet + return h.lower(), r, t.lower() class UpdatedWebQSPDataset(WebQSPDataset): @@ -103,8 +104,8 @@ def _retrieve_subgraphs(self) -> None: graph = get_features_for_triplets( self.indexer, local_trips, pre_transform=preprocess_triplet ) - textual_nodes = self.textual_nodes.iloc[graph["node_idx"]] - textual_edges = self.textual_edges.iloc[graph["edge_idx"]] + textual_nodes = self.textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = self.textual_edges.iloc[graph["edge_idx"]].reset_index() self.raw_graphs.append(graph) pcst_subgraph, desc = retrieval_via_pcst( graph, From 168a37a5be66bb81903dc27c49386f803e216686 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 31 May 2024 17:17:42 -0700 Subject: [PATCH 590/752] fixes after rebase --- examples/llm_plus_gnn/raw_qsp_dataset.py | 16 ++++++---------- examples/llm_plus_gnn/updated_qsp_dataset.py | 10 ++++------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/examples/llm_plus_gnn/raw_qsp_dataset.py b/examples/llm_plus_gnn/raw_qsp_dataset.py index d8a3cf0a64e0..afa4fc121b51 100644 --- a/examples/llm_plus_gnn/raw_qsp_dataset.py +++ b/examples/llm_plus_gnn/raw_qsp_dataset.py @@ -55,16 +55,14 @@ def download(self) -> None: def process(self) -> None: if self.with_process: - pretrained_repo = "sentence-transformers/all-roberta-large-v1" - self.model = SentenceTransformer(pretrained_repo) - self.model.to(self.device) + self.model = SentenceTransformer().to(self.device) self.model.eval() list_of_graphs = [] self.raw_graphs = [] if self.with_pcst: self.questions = [i["question"] for i in self.raw_dataset] print("Encoding questions...") - q_embs = text2embedding(self.model, self.device, self.questions) + q_embs = self.model.encode(self.questions, batch_size=256) print("Encoding graphs...") limit = self.limit if self.limit else len(self.raw_dataset) @@ -83,18 +81,16 @@ def process(self) -> None: raw_edges.append( {"src": raw_nodes[h], "edge_attr": r, "dst": raw_nodes[t]} ) - nodes = pd.DataFrame( + nodes = DataFrame( [{"node_id": v, "node_attr": k} for k, v in raw_nodes.items()], columns=["node_id", "node_attr"], ) - edges = pd.DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) + edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") - x = text2embedding(self.model, self.device, nodes.node_attr.tolist()) + x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) # encode edges - edge_attr = text2embedding( - self.model, self.device, edges.edge_attr.tolist() - ) + edge_attr = self.model.encode(edges.edge_attr.tolist(), batch_size=256) edge_index = torch.LongTensor([edges.src.tolist(), edges.dst.tolist()]) question = f"Question: {data_i['question']}\nAnswer: " label = ("|").join(data_i["answer"]).lower() diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py index e5e5de84338d..2b1e1c1ceb8a 100644 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -70,12 +70,12 @@ def _build_graph(self) -> None: # Nodes: nodes = self.indexer.get_unique_node_features() - x = text2embedding(self.model, self.device, nodes) + x = self.model.encode(nodes, batch_size=256) self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: edges = self.indexer.get_unique_edge_features(feature_name=EDGE_RELATION) - edge_attr = text2embedding(self.model, self.device, edges) + edge_attr = self.model.encode(edges, batch_size=256) self.indexer.add_edge_feature( new_feature_name="edge_attr", new_feature_vals=edge_attr, @@ -87,7 +87,7 @@ def _build_graph(self) -> None: def _retrieve_subgraphs(self) -> None: print("Encoding questions...") self.questions = [ds["question"] for ds in self.raw_dataset] - q_embs = text2embedding(self.model, self.device, self.questions) + q_embs = self.model.encode(self.questions, batch_size=256) list_of_graphs = [] self.raw_graphs = [] print("Retrieving subgraphs...") @@ -128,9 +128,7 @@ def _retrieve_subgraphs(self) -> None: def process(self) -> None: self._load_raw_data() - pretrained_repo = "sentence-transformers/all-roberta-large-v1" - self.model = SentenceTransformer(pretrained_repo) - self.model.to(self.device) + self.model = SentenceTransformer().to(self.device) self.model.eval() if not os.path.exists(self.processed_dir[-1]): print("Encoding graph...") From 2cf727f154d9e8cc670cf4a6a98c954b7e469961 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 31 May 2024 19:12:37 -0700 Subject: [PATCH 591/752] more fixes and mocking --- .../test_eval_large_graph_indexer.ipynb | 48 ++-- .../test_eval_updated_webqsp.ipynb | 222 +++++++++++------- examples/llm_plus_gnn/updated_qsp_dataset.py | 4 +- 3 files changed, 155 insertions(+), 119 deletions(-) diff --git a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb index 80220ddcf7ab..e3448a0231d4 100644 --- a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb +++ b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb @@ -21,7 +21,8 @@ "from torch_geometric.datasets.web_qsp_dataset import *\n", "import torch\n", "import datasets\n", - "import time" + "import time\n", + "from unittest.mock import patch" ] }, { @@ -90,7 +91,8 @@ "outputs": [], "source": [ "def preprocess_triplet(triplet: TripletLike):\n", - " return tuple([word.lower() for word in triplet])" + " h, r, t = triplet\n", + " return h.lower(), r, t.lower()" ] }, { @@ -200,7 +202,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 100/100 [00:01<00:00, 51.52it/s]\n" + "100%|██████████| 100/100 [00:01<00:00, 55.69it/s]\n" ] } ], @@ -220,7 +222,7 @@ { "data": { "text/plain": [ - "0.6365907192230225" + "0.6796014308929443" ] }, "execution_count": 16, @@ -280,7 +282,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "423763it [00:02, 198021.34it/s]\n" + "423763it [00:02, 206877.03it/s]\n" ] } ], @@ -496,7 +498,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "3295it [01:07, 48.79it/s] \n" + "3295it [01:05, 50.02it/s] \n" ] } ], @@ -559,7 +561,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "97it [00:01, 51.39it/s] \n" + "97it [00:01, 50.34it/s] \n" ] } ], @@ -636,7 +638,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [00:01<00:00, 6.19it/s]\n" + "100%|██████████| 10/10 [00:01<00:00, 6.60it/s]\n" ] } ], @@ -661,7 +663,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "inherit model weights from sentence-transformers/all-roberta-large-v1\n", "Encoding graphs...\n" ] }, @@ -669,7 +670,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [01:00<00:00, 6.07s/it]\n", + "100%|██████████| 10/10 [00:00<00:00, 56.01it/s]\n", "Done!\n" ] } @@ -681,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -697,26 +698,9 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n" - ] - } - ], + "outputs": [], "source": [ "for ds in zip(old_dataset, first_10):\n", " print(results_are_close_enough(*ds))" @@ -731,7 +715,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -740,7 +724,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb index 0fc7f525158f..e989866f308e 100644 --- a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb +++ b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb @@ -16,12 +16,13 @@ ], "source": [ "from updated_qsp_dataset import UpdatedWebQSPDataset\n", - "from raw_qsp_dataset import RawWebQSPDataset" + "from raw_qsp_dataset import RawWebQSPDataset\n", + "from unittest.mock import patch" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -35,7 +36,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "inherit model weights from sentence-transformers/all-roberta-large-v1\n", "Encoding questions...\n", "Encoding graphs...\n" ] @@ -44,7 +44,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 2/2 [00:08<00:00, 4.30s/it]\n", + "100%|██████████| 2/2 [00:07<00:00, 3.63s/it]\n", "Done!\n", "Processing...\n" ] @@ -53,7 +53,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "inherit model weights from sentence-transformers/all-roberta-large-v1\n", "Encoding graph...\n", "Encoding questions...\n", "Retrieving subgraphs...\n" @@ -63,84 +62,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 2/2 [00:00<00:00, 3.56it/s]\n", + "100%|██████████| 2/2 [00:00<00:00, 3.69it/s]\n", "Done!\n" ] } ], "source": [ + "old_mock, new_mock = None, None\n", "old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", - "new_dataset = UpdatedWebQSPDataset(root='updated_dataset',force_reload=True, limit=2)" + "new_dataset = UpdatedWebQSPDataset(root='updated_dataset',force_reload=True, limit=2, whole_graph_retrieval=False)" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Data(x=[15, 1024], edge_index=[2, 20], edge_attr=[20, 1024], question='Question: what is the name of justin bieber brother\n", - "Answer: ', label='jaxon bieber', desc='node_id,node_attr\n", - "249,m.0129jzth\n", - "434,jeremy bieber\n", - "880,m.0gxnp5d\n", - "892,m.0gxnnwc\n", - "1178,justin bieber\n", - "1239,yves bole\n", - "1563,english language\n", - "1569,jazmyn bieber\n", - "1742,pattie mallette\n", - "1858,m.0gxnp5m\n", - "1888,this is justin bieber\n", - "2050,jaxon bieber\n", - "2205,m.0gxnnwp\n", - "2436,m.0wfn4pm\n", - "2620,m.0gxnp5x\n", - "\n", - "src,edge_attr,dst\n", - "1239,people.person.languages,1563\n", - "1888,film.film.language,1563\n", - "1239,influence.influence_node.influenced_by,1178\n", - "1742,people.person.children,1178\n", - "2050,people.person.parents,434\n", - "2436,people.sibling_relationship.sibling,1742\n", - "2050,people.person.sibling_s,2205\n", - "1178,base.popstra.celebrity.hangout,2620\n", - "892,people.sibling_relationship.sibling,1569\n", - "249,people.sibling_relationship.sibling,1239\n", - "1742,people.person.sibling_s,2436\n", - "2205,people.sibling_relationship.sibling,2050\n", - "1239,people.person.sibling_s,249\n", - "1178,people.person.sibling_s,892\n", - "1178,base.popstra.celebrity.hangout,880\n", - "1569,people.person.sibling_s,892\n", - "892,people.sibling_relationship.sibling,1178\n", - "1178,people.person.sibling_s,2205\n", - "1178,base.popstra.celebrity.hangout,1858\n", - "2205,people.sibling_relationship.sibling,1178\n", - "', num_nodes=15)" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_dataset[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Data(x=[12, 1024], edge_index=[2, 13], edge_attr=[13, 1024], question='Question: what is the name of justin bieber brother\n", + "Data(x=[12, 1024], edge_index=[2, 12], edge_attr=[12, 1024], question='Question: what is the name of justin bieber brother\n", "Answer: ', label='jaxon bieber', desc='node_id,node_attr\n", "15,justin bieber\n", "151,pattie mallette\n", @@ -161,18 +102,17 @@ "346,influence.influence_node.influenced_by,15\n", "151,people.person.children,15\n", "294,people.person.parents,356\n", - "294,people.person.sibling_s,551\n", - "452,people.person.sibling_s,933\n", "545,people.sibling_relationship.sibling,151\n", "933,people.sibling_relationship.sibling,452\n", "1359,people.sibling_relationship.sibling,346\n", "551,people.sibling_relationship.sibling,294\n", + "15,people.person.sibling_s,933\n", "933,people.sibling_relationship.sibling,15\n", "551,people.sibling_relationship.sibling,15\n", "', num_nodes=12)" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -183,25 +123,106 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Data(x=[15, 1024], edge_index=[2, 20], edge_attr=[20, 1024], question='Question: what is the name of justin bieber brother\n", + "Answer: ', label='jaxon bieber', desc='index,node_id,node_attr\n", + "54,54,m.0gxnp5m\n", + "307,307,m.0129jzth\n", + "391,391,pattie mallette\n", + "650,650,english language\n", + "837,837,justin bieber\n", + "934,934,this is justin bieber\n", + "1022,1022,m.0gxnp5x\n", + "1123,1123,jeremy bieber\n", + "1149,1149,m.0gxnnwp\n", + "1847,1847,m.0gxnnwc\n", + "2294,2294,yves bole\n", + "2702,2702,m.0gxnp5d\n", + "2719,2719,jazmyn bieber\n", + "2880,2880,m.0wfn4pm\n", + "2882,2882,jaxon bieber\n", + "\n", + "src,edge_attr,dst\n", + "2294,people.person.languages,650\n", + "934,film.film.language,650\n", + "2294,influence.influence_node.influenced_by,837\n", + "391,people.person.children,837\n", + "2882,people.person.parents,1123\n", + "2880,people.sibling_relationship.sibling,391\n", + "2882,people.person.sibling_s,1149\n", + "837,base.popstra.celebrity.hangout,1022\n", + "1847,people.sibling_relationship.sibling,2719\n", + "307,people.sibling_relationship.sibling,2294\n", + "391,people.person.sibling_s,2880\n", + "1149,people.sibling_relationship.sibling,2882\n", + "2294,people.person.sibling_s,307\n", + "837,people.person.sibling_s,1847\n", + "837,base.popstra.celebrity.hangout,2702\n", + "2719,people.person.sibling_s,1847\n", + "1847,people.sibling_relationship.sibling,837\n", + "837,people.person.sibling_s,1149\n", + "837,base.popstra.celebrity.hangout,54\n", + "1149,people.sibling_relationship.sibling,837\n", + "', num_nodes=15)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "new_dataset.raw_graphs[0]" + "new_dataset[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Data(x=[1709, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], num_nodes=1709)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "old_dataset.raw_graphs[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Data(x=[1709, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], num_nodes=1709, pid=[1709], e_pid=[9088], node_idx=[1709], edge_idx=[9088])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "new_dataset.raw_graphs[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -212,25 +233,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ - "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", + "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.9):\n", " def _sorted_tensors_are_close(tensor1, tensor2):\n", - " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", + " vals = torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1)\n", + " return torch.all(vals > thresh)\n", " def _graphs_are_same(tensor1, tensor2):\n", " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", - " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", - " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", - " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" + " val = _sorted_tensors_are_close(ground_truth.x, new_method.x)\n", + " val &= _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr)\n", + " val &= _graphs_are_same(ground_truth.edge_index, new_method.edge_index)\n", + " return val" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(False)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "results_are_close_enough(old_dataset.raw_graphs[0], new_dataset.raw_graphs[0])" ] @@ -253,6 +287,24 @@ "new_dataset.questions[0]" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "torch.sort(new_dataset.raw_graphs[0].edge_attr, 0)[0].unique(dim=0).size(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "torch.sort(old_dataset.raw_graphs[0].edge_attr, 0)[0].unique(dim=0).size()" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py index 2b1e1c1ceb8a..c40c7fe26a43 100644 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -136,12 +136,12 @@ def process(self) -> None: else: print("Loading graph...") self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) - self.textual_nodes = pd.DataFrame.from_dict( + self.textual_nodes = DataFrame.from_dict( {"node_attr": self.indexer.get_node_features()} ) self.textual_nodes["node_id"] = self.textual_nodes.index self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] - self.textual_edges = pd.DataFrame( + self.textual_edges = DataFrame( self.indexer.get_edge_features(), columns=["src", "edge_attr", "dst"] ) self.textual_edges["src"] = [ From 9ab9b221eeeb9b530c3bc8d389b77b9db1529e85 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 31 May 2024 19:47:38 -0700 Subject: [PATCH 592/752] more mocking --- .../test_eval_updated_webqsp.ipynb | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb index e989866f308e..f067c4f5ae58 100644 --- a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb +++ b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb @@ -17,7 +17,30 @@ "source": [ "from updated_qsp_dataset import UpdatedWebQSPDataset\n", "from raw_qsp_dataset import RawWebQSPDataset\n", - "from unittest.mock import patch" + "from unittest.mock import patch\n", + "from torch_geometric.data import Data\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def tokenizer_call_iter():\n", + " a, batch_size = [], 256\n", + " i = 0\n", + " while True:\n", + " rv = torch.Tensor([list(range(i, i+len(a)))])\n", + " i += len(a)\n", + " a = yield rv\n", + " if a is None:\n", + " a = []\n", + "gen = tokenizer_call_iter()\n", + "next(gen)\n", + "def tokenizer_call(a, batch_size):\n", + " return gen.send(a)" ] }, { @@ -44,32 +67,45 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 2/2 [00:07<00:00, 3.63s/it]\n", - "Done!\n", - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding graph...\n", - "Encoding questions...\n", - "Retrieving subgraphs...\n" + " 0%| | 0/2 [00:00 6\u001b[0m old_dataset \u001b[38;5;241m=\u001b[39m \u001b[43mRawWebQSPDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mold_dataset\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwith_process\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwith_pcst\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m old_mock \u001b[38;5;241m=\u001b[39m model_mock\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/raw_qsp_dataset.py:20\u001b[0m, in \u001b[0;36mRawWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, with_process, with_pcst, limit)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlimit \u001b[38;5;241m=\u001b[39m limit\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwith_process:\n\u001b[0;32m---> 20\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_dependencies()\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/web_qsp_dataset.py:180\u001b[0m, in \u001b[0;36mWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_dependencies()\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 180\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/raw_qsp_dataset.py:105\u001b[0m, in \u001b[0;36mRawWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw_graphs\u001b[38;5;241m.\u001b[39mappend(raw_graph)\n\u001b[1;32m 104\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwith_pcst:\n\u001b[0;32m--> 105\u001b[0m psct_subgraph, desc \u001b[38;5;241m=\u001b[39m \u001b[43mretrieval_via_pcst\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 106\u001b[0m \u001b[43m \u001b[49m\u001b[43mraw_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 107\u001b[0m \u001b[43m \u001b[49m\u001b[43mq_embs\u001b[49m\u001b[43m[\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 108\u001b[0m \u001b[43m \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 109\u001b[0m \u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 110\u001b[0m \u001b[43m \u001b[49m\u001b[43mtopk\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 111\u001b[0m \u001b[43m \u001b[49m\u001b[43mtopk_e\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[43m \u001b[49m\u001b[43mcost_e\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 113\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 114\u001b[0m psct_subgraph[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquestion\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m question\n\u001b[1;32m 115\u001b[0m psct_subgraph[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m label\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/web_qsp_dataset.py:66\u001b[0m, in \u001b[0;36mretrieval_via_pcst\u001b[0;34m(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e)\u001b[0m\n\u001b[1;32m 64\u001b[0m n_prizes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mCosineSimilarity(dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)(q_emb, graph\u001b[38;5;241m.\u001b[39mx)\n\u001b[1;32m 65\u001b[0m topk \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmin\u001b[39m(topk, graph\u001b[38;5;241m.\u001b[39mnum_nodes)\n\u001b[0;32m---> 66\u001b[0m _, topk_n_indices \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtopk\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn_prizes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtopk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlargest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 68\u001b[0m n_prizes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mzeros_like(n_prizes)\n\u001b[1;32m 69\u001b[0m n_prizes[topk_n_indices] \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39marange(topk, \u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mfloat()\n", + "\u001b[0;31mRuntimeError\u001b[0m: selected index k out of range" ] } ], "source": [ - "old_mock, new_mock = None, None\n", - "old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", + "token_mock, ret_mock = None, None\n", + "with patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock:\n", + " with patch(\"torch_geometric.datasets.web_qsp_dataset.retrieval_via_pcst\") as pcst_mock:\n", + " model_mock.side_effect =gen\n", + " pcst_mock.return_value = (Data(), \"\")\n", + " old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", + " token_mock = model_mock\n", + " ret_mock = pcst_mock" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "new_dataset = UpdatedWebQSPDataset(root='updated_dataset',force_reload=True, limit=2, whole_graph_retrieval=False)" ] }, From 6a1ee0ff70d5698fdabe2ed15f5a2782a37b5f67 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 15:19:21 -0700 Subject: [PATCH 593/752] main mocking working --- .../test_eval_updated_webqsp.ipynb | 240 ++++++++++++++++-- 1 file changed, 213 insertions(+), 27 deletions(-) diff --git a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb index f067c4f5ae58..59480bbc5463 100644 --- a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb +++ b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ " a, batch_size = [], 256\n", " i = 0\n", " while True:\n", - " rv = torch.Tensor([list(range(i, i+len(a)))])\n", + " rv = torch.Tensor([list(range(i, i+len(a)))]).T\n", " i += len(a)\n", " a = yield rv\n", " if a is None:\n", @@ -45,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -67,37 +67,223 @@ "name": "stderr", "output_type": "stream", "text": [ - " 0%| | 0/2 [00:00 6\u001b[0m old_dataset \u001b[38;5;241m=\u001b[39m \u001b[43mRawWebQSPDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mold_dataset\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwith_process\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwith_pcst\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m old_mock \u001b[38;5;241m=\u001b[39m model_mock\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/raw_qsp_dataset.py:20\u001b[0m, in \u001b[0;36mRawWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, with_process, with_pcst, limit)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlimit \u001b[38;5;241m=\u001b[39m limit\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwith_process:\n\u001b[0;32m---> 20\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_dependencies()\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/web_qsp_dataset.py:180\u001b[0m, in \u001b[0;36mWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_dependencies()\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 180\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/raw_qsp_dataset.py:105\u001b[0m, in \u001b[0;36mRawWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw_graphs\u001b[38;5;241m.\u001b[39mappend(raw_graph)\n\u001b[1;32m 104\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwith_pcst:\n\u001b[0;32m--> 105\u001b[0m psct_subgraph, desc \u001b[38;5;241m=\u001b[39m \u001b[43mretrieval_via_pcst\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 106\u001b[0m \u001b[43m \u001b[49m\u001b[43mraw_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 107\u001b[0m \u001b[43m \u001b[49m\u001b[43mq_embs\u001b[49m\u001b[43m[\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 108\u001b[0m \u001b[43m \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 109\u001b[0m \u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 110\u001b[0m \u001b[43m \u001b[49m\u001b[43mtopk\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 111\u001b[0m \u001b[43m \u001b[49m\u001b[43mtopk_e\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[43m \u001b[49m\u001b[43mcost_e\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 113\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 114\u001b[0m psct_subgraph[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquestion\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m question\n\u001b[1;32m 115\u001b[0m psct_subgraph[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m label\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/web_qsp_dataset.py:66\u001b[0m, in \u001b[0;36mretrieval_via_pcst\u001b[0;34m(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e)\u001b[0m\n\u001b[1;32m 64\u001b[0m n_prizes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mCosineSimilarity(dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)(q_emb, graph\u001b[38;5;241m.\u001b[39mx)\n\u001b[1;32m 65\u001b[0m topk \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmin\u001b[39m(topk, graph\u001b[38;5;241m.\u001b[39mnum_nodes)\n\u001b[0;32m---> 66\u001b[0m _, topk_n_indices \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtopk\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn_prizes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtopk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlargest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 68\u001b[0m n_prizes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mzeros_like(n_prizes)\n\u001b[1;32m 69\u001b[0m n_prizes[topk_n_indices] \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39marange(topk, \u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mfloat()\n", - "\u001b[0;31mRuntimeError\u001b[0m: selected index k out of range" + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 2/2 [00:00<00:00, 6.98it/s]\n", + "Done!\n" ] } ], "source": [ - "token_mock, ret_mock = None, None\n", - "with patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock:\n", - " with patch(\"torch_geometric.datasets.web_qsp_dataset.retrieval_via_pcst\") as pcst_mock:\n", - " model_mock.side_effect =gen\n", - " pcst_mock.return_value = (Data(), \"\")\n", - " old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", - " token_mock = model_mock\n", - " ret_mock = pcst_mock" + "new_token_mock, new_ret_mock = None, None\n", + "with (patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock,\n", + " patch(\"updated_qsp_dataset.retrieval_via_pcst\") as pcst_mock):\n", + " model_mock.side_effect = tokenizer_from_map\n", + " pcst_mock.return_value = Data(), \"\"\n", + " old_dataset = UpdatedWebQSPDataset(root='old_dataset', force_reload=True, limit=2, whole_graph_retrieval=False)\n", + " new_token_mock = model_mock\n", + " new_ret_mock = pcst_mock" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "old_nodes_called = [old_ret_mock.call_args_list[i][0][0].x.int().numpy() for i in range(len(old_ret_mock.call_args_list))]\n", + "old_edges_called = [old_ret_mock.call_args_list[i][0][0].edge_attr.int().numpy() for i in range(len(old_ret_mock.call_args_list))]\n", + "old_question_called = [old_ret_mock.call_args_list[i][0][1].int().numpy() for i in range(len(old_ret_mock.call_args_list))]" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "new_nodes_called = [new_ret_mock.call_args_list[i][0][0].x.int().numpy() for i in range(len(new_ret_mock.call_args_list))]\n", + "new_edges_called = [new_ret_mock.call_args_list[i][0][0].edge_attr.int().numpy() for i in range(len(new_ret_mock.call_args_list))]\n", + "new_question_called = [new_ret_mock.call_args_list[i][0][1].int().numpy() for i in range(len(new_ret_mock.call_args_list))]" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n", + "True\n", + "True\n", + "True\n", + "True\n" + ] + } + ], + "source": [ + "for i in range(len(old_ret_mock.call_args_list)):\n", + " print(set(old_nodes_called[i]) == set(new_nodes_called[i]))\n", + " print(set(old_edges_called[i]) == set(new_edges_called[i]))\n", + " print(old_question_called[i] == new_question_called[i])" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array(3073, dtype=int32), array(2115, dtype=int32)]" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "old_question_called" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([ 0., 3971., 3972., ..., 8044., 8046., 5478.]),\n", + " tensor([ 12., 3985., 3987., ..., 3953., 8035., 8038.])]" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "new_nodes_called" ] }, { From 03f6f4ee8a835592b085980bc8011dcc5f790b2f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 15:19:37 -0700 Subject: [PATCH 594/752] pr fixes --- .../test_eval_large_graph_indexer.ipynb | 87 ++++++------------- 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb index e3448a0231d4..0b6ed8b5635e 100644 --- a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb +++ b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb @@ -367,81 +367,32 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "from torch_geometric.nn.text import text2embedding, SentenceTransformer" + "from torch_geometric.nn.nlp import SentenceTransformer" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 48, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "inherit model weights from sentence-transformers/all-roberta-large-v1\n" - ] - }, { "data": { "text/plain": [ - "SentenceTransformer(\n", - " (bert_model): RobertaModel(\n", - " (embeddings): RobertaEmbeddings(\n", - " (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n", - " (position_embeddings): Embedding(514, 1024, padding_idx=1)\n", - " (token_type_embeddings): Embedding(1, 1024)\n", - " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): RobertaEncoder(\n", - " (layer): ModuleList(\n", - " (0-23): 24 x RobertaLayer(\n", - " (attention): RobertaAttention(\n", - " (self): RobertaSelfAttention(\n", - " (query): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (key): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (value): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): RobertaSelfOutput(\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): RobertaIntermediate(\n", - " (dense): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): RobertaOutput(\n", - " (dense): Linear(in_features=4096, out_features=1024, bias=True)\n", - " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): RobertaPooler(\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " )\n", - ")" + "SentenceTransformer(model_name=sentence-transformers/all-roberta-large-v1)" ] }, - "execution_count": 27, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(\"sentence-transformers/all-roberta-large-v1\").to(device)\n", + "model = SentenceTransformer().to(device)\n", "model.eval()" ] }, @@ -491,14 +442,26 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "3295it [01:05, 50.02it/s] \n" + " 0%| | 0/3294 [00:00 4\u001b[0m node_embs\u001b[38;5;241m.\u001b[39mappend(\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m(nbatch, batch_size\u001b[38;5;241m=\u001b[39mBATCH_SIZE)\u001b[38;5;241m.\u001b[39mcpu())\n\u001b[1;32m 5\u001b[0m node_embs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat(node_embs, \u001b[38;5;241m0\u001b[39m)\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/torch/nn/modules/module.py:1709\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1708\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1709\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mAttributeError\u001b[0m: 'SentenceTransformer' object has no attribute 'encode'" ] } ], @@ -506,7 +469,7 @@ "# Indexing graph features\n", "node_embs = []\n", "for nbatch in tqdm.tqdm(chunked(node_attributes, BATCH_SIZE), total=nodes_to_process//BATCH_SIZE):\n", - " node_embs.append(text2embedding(model, device, nbatch, BATCH_SIZE).cpu())\n", + " node_embs.append(model.encode(nbatch, batch_size=BATCH_SIZE).cpu())\n", "node_embs = torch.cat(node_embs, 0)\n", "\n" ] @@ -569,7 +532,7 @@ "# Indexing graph features\n", "edge_embs = []\n", "for ebatch in tqdm.tqdm(chunked(edge_attributes, BATCH_SIZE), total=edges_to_process//BATCH_SIZE):\n", - " edge_embs.append(text2embedding(model, device, ebatch).cpu())\n", + " edge_embs.append(model.encode(ebatch, batch_size=BATCH_SIZE).cpu())\n", "edge_embs = torch.cat(edge_embs, 0)\n" ] }, @@ -677,7 +640,11 @@ ], "source": [ "# Grab the first few samples from the old ds to test with LargeGraphIndexer\n", - "old_dataset = RawWebQSPDataset(force_reload=True, with_process=True, limit=10)" + "raw_mock = None\n", + "with patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as mock_method:\n", + " mock_method.return_value: torch.Tensor = torch.zeros(256, 1024)\n", + "old_dataset = RawWebQSPDataset(force_reload=True, with_process=True, limit=10)\n", + " raw_mock = mock_method" ] }, { From 0b4ee3d3a95eee23915a4616af97961330954610 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 15:36:11 -0700 Subject: [PATCH 595/752] migrate large graph indexer --- .../llm_plus_gnn => torch_geometric/data}/large_graph_indexer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {examples/llm_plus_gnn => torch_geometric/data}/large_graph_indexer.py (100%) diff --git a/examples/llm_plus_gnn/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py similarity index 100% rename from examples/llm_plus_gnn/large_graph_indexer.py rename to torch_geometric/data/large_graph_indexer.py From 81a1b8afba2c49ef53c052bbb8c6111c51dff746 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 16:04:19 -0700 Subject: [PATCH 596/752] fix save overrider --- torch_geometric/data/large_graph_indexer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 1a9151d26594..b84a307f2270 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -16,6 +16,7 @@ Tuple, Type, ) +import shutil import torch from torch_geometric.data import Data, FeatureStore, GraphStore @@ -331,6 +332,8 @@ def to_triplets(self) -> Iterator[TripletLike]: return iter(self.edge_attr[EDGE_PID]) def save(self, path: str) -> None: + if os.path.exists(path): + shutil.rmtree(path) os.makedirs(path, exist_ok=True) with open(path + "/edges", "wb") as f: pkl.dump(self._edges, f) From dd2ec5623665fa9628c6e3fab6d72b667fe3360f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 16:43:41 -0700 Subject: [PATCH 597/752] formatting 1 --- torch_geometric/data/large_graph_indexer.py | 44 +++++++++++++-------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index b84a307f2270..a763026f6b53 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -1,5 +1,6 @@ import os import pickle as pkl +import shutil from dataclasses import dataclass from itertools import chain from typing import ( @@ -14,13 +15,11 @@ Sequence, Set, Tuple, - Type, ) -import shutil import torch -from torch_geometric.data import Data, FeatureStore, GraphStore -from torch_geometric.distributed import LocalFeatureStore, LocalGraphStore + +from torch_geometric.data import Data # Is there a multiprocessing-friendly implementation of this? @@ -89,10 +88,11 @@ def __init__( self.node_attr = node_attr if NODE_KEYS & set(self.node_attr.keys()) != NODE_KEYS: raise AttributeError( - f"Invalid node_attr object. Missing {NODE_KEYS - set(self.node_attr.keys())}" + "Invalid node_attr object. Missing " + + f"{NODE_KEYS - set(self.node_attr.keys())}" ) elif self.node_attr[NODE_PID] != nodes: - raise AttributeError(f"Nodes provided do not match those in node_attr") + raise AttributeError("Nodes provided do not match those in node_attr") else: self.node_attr = dict() self.node_attr[NODE_PID] = nodes @@ -106,10 +106,11 @@ def __init__( if EDGE_KEYS & set(self.edge_attr.keys()) != EDGE_KEYS: raise AttributeError( - f"Invalid edge_attr object. Missing {EDGE_KEYS - set(self.edge_attr.keys())}" + "Invalid edge_attr object. Missing " + + f"{EDGE_KEYS - set(self.edge_attr.keys())}" ) elif self.node_attr[EDGE_PID] != edges: - raise AttributeError(f"Edges provided do not match those in edge_attr") + raise AttributeError("Edges provided do not match those in edge_attr") else: self.edge_attr = dict() @@ -139,7 +140,9 @@ def from_triplets( if pre_transform is not None: - def apply_transform(trips: Iterable[TripletLike]) -> Iterator[TripletLike]: + def apply_transform( + trips: Iterable[TripletLike], + ) -> Iterator[TripletLike]: for trip in trips: yield pre_transform(trip) @@ -164,7 +167,7 @@ def collate(cls, graphs: Iterable["LargeGraphIndexer"]) -> "LargeGraphIndexer": def get_unique_node_features(self, feature_name: str = NODE_PID) -> List[Hashable]: try: if feature_name in self._mapped_node_features: - raise IndexError(f"Only non-mapped features can be retrieved uniquely.") + raise IndexError("Only non-mapped features can be retrieved uniquely.") return ordered_set(self.get_node_features(feature_name)) except KeyError: @@ -185,7 +188,8 @@ def add_node_feature( feature_keys = self.get_unique_node_features(map_from_feature) if len(feature_keys) != len(new_feature_vals): raise AttributeError( - f"Expected encodings for {len(feature_keys)} unique features, but got {len(new_feature_vals)} encodings." + "Expected encodings for {len(feature_keys)} unique features," + + f" but got {len(new_feature_vals)} encodings." ) if map_from_feature == NODE_PID: @@ -197,7 +201,9 @@ def add_node_feature( self._mapped_node_features.add(new_feature_name) def get_node_features( - self, feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None + self, + feature_name: str = NODE_PID, + pids: Optional[Iterable[Hashable]] = None, ) -> List[Any]: if feature_name in self._mapped_node_features: values = self.node_attr[feature_name].values @@ -247,7 +253,7 @@ def get_node_features_iter( def get_unique_edge_features(self, feature_name: str = EDGE_PID) -> List[Hashable]: try: if feature_name in self._mapped_edge_features: - raise IndexError(f"Only non-mapped features can be retrieved uniquely.") + raise IndexError("Only non-mapped features can be retrieved uniquely.") return ordered_set(self.get_edge_features(feature_name)) except KeyError: raise AttributeError(f"Edges do not have a feature called {feature_name}") @@ -267,7 +273,8 @@ def add_edge_feature( feature_keys = self.get_unique_edge_features(map_from_feature) if len(feature_keys) != len(new_feature_vals): raise AttributeError( - f"Expected encodings for {len(feature_keys)} unique features, but got {len(new_feature_vals)} encodings." + "Expected encodings for {len(feature_keys)} unique features, " + + f"but got {len(new_feature_vals)} encodings." ) if map_from_feature == EDGE_PID: @@ -392,13 +399,13 @@ def __eq__(self, value: "LargeGraphIndexer") -> bool: eq &= self._mapped_edge_features == value._mapped_edge_features for k in self.node_attr: - eq &= type(self.node_attr[k]) == type(value.node_attr[k]) + eq &= isinstance(self.node_attr[k], type(value.node_attr[k])) if isinstance(self.node_attr[k], torch.Tensor): eq &= torch.equal(self.node_attr[k], value.node_attr[k]) else: eq &= self.node_attr[k] == value.node_attr[k] for k in self.edge_attr: - eq &= type(self.edge_attr[k]) == type(value.edge_attr[k]) + eq &= isinstance(self.edge_attr[k], type(value.edge_attr[k])) if isinstance(self.edge_attr[k], torch.Tensor): eq &= torch.equal(self.edge_attr[k], value.edge_attr[k]) else: @@ -462,7 +469,10 @@ def apply_transform(trips): edge_attr = torch.Tensor(edge_feats) edge_index = torch.t(torch.LongTensor(edge_index)) data_obj = Data( - x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(node_feats) + x=x, + edge_index=edge_index, + edge_attr=edge_attr, + num_nodes=len(node_feats), ) # needed for mappings data_obj[NODE_PID] = node_keys From 917fe549efd7c968c6864020993f78b1e22e9971 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 16:55:20 -0700 Subject: [PATCH 598/752] formatting 2 --- examples/llm_plus_gnn/raw_qsp_dataset.py | 46 +++++--- examples/llm_plus_gnn/updated_qsp_dataset.py | 46 ++++---- torch_geometric/data/large_graph_indexer.py | 113 +++++++++---------- torch_geometric/datasets/web_qsp_dataset.py | 49 ++++---- 4 files changed, 136 insertions(+), 118 deletions(-) diff --git a/examples/llm_plus_gnn/raw_qsp_dataset.py b/examples/llm_plus_gnn/raw_qsp_dataset.py index afa4fc121b51..cd6a16e8aed2 100644 --- a/examples/llm_plus_gnn/raw_qsp_dataset.py +++ b/examples/llm_plus_gnn/raw_qsp_dataset.py @@ -1,10 +1,19 @@ -from typing import Optional +from typing import Dict, List, Optional -from torch_geometric.datasets.web_qsp_dataset import * +import torch +from tqdm import tqdm +from torch_geometric.data import Data, InMemoryDataset +from torch_geometric.datasets.web_qsp_dataset import ( + DataFrame, + WebQSPDataset, + datasets, + retrieval_via_pcst, +) +from torch_geometric.nn.nlp import SentenceTransformer -class RawWebQSPDataset(WebQSPDataset): +class RawWebQSPDataset(WebQSPDataset): def __init__( self, root: str = "", @@ -20,10 +29,10 @@ def __init__( super().__init__(root, force_reload) else: self._check_dependencies() - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - super(InMemoryDataset, self).__init__( - root, None, None, force_reload=force_reload - ) + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + super(InMemoryDataset, self).__init__(root, None, None, + force_reload=force_reload) self._load_raw_data() @property @@ -78,20 +87,29 @@ def process(self) -> None: raw_nodes[h] = len(raw_nodes) if t not in raw_nodes: raw_nodes[t] = len(raw_nodes) - raw_edges.append( - {"src": raw_nodes[h], "edge_attr": r, "dst": raw_nodes[t]} - ) + raw_edges.append({ + "src": raw_nodes[h], + "edge_attr": r, + "dst": raw_nodes[t] + }) nodes = DataFrame( - [{"node_id": v, "node_attr": k} for k, v in raw_nodes.items()], + [{ + "node_id": v, + "node_attr": k + } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"], ) - edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) + edges = DataFrame(raw_edges, + columns=["src", "edge_attr", "dst"]) # encode nodes nodes.node_attr = nodes.node_attr.fillna("") x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) # encode edges - edge_attr = self.model.encode(edges.edge_attr.tolist(), batch_size=256) - edge_index = torch.LongTensor([edges.src.tolist(), edges.dst.tolist()]) + edge_attr = self.model.encode(edges.edge_attr.tolist(), + batch_size=256) + edge_index = torch.LongTensor( + [edges.src.tolist(), + edges.dst.tolist()]) question = f"Question: {data_i['question']}\nAnswer: " label = ("|").join(data_i["answer"]).lower() raw_graph = Data( diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py index c40c7fe26a43..97e519d64056 100644 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ b/examples/llm_plus_gnn/updated_qsp_dataset.py @@ -1,14 +1,23 @@ import os from itertools import chain -from typing import Iterator, Optional +from typing import Iterator, List +import torch from large_graph_indexer import ( EDGE_RELATION, LargeGraphIndexer, TripletLike, get_features_for_triplets, ) -from torch_geometric.datasets.web_qsp_dataset import * +from tqdm import tqdm + +from torch_geometric.datasets.web_qsp_dataset import ( + DataFrame, + WebQSPDataset, + datasets, + retrieval_via_pcst, +) +from torch_geometric.nn.nlp import SentenceTransformer def preprocess_triplet(triplet: TripletLike) -> TripletLike: @@ -17,7 +26,6 @@ def preprocess_triplet(triplet: TripletLike) -> TripletLike: class UpdatedWebQSPDataset(WebQSPDataset): - def __init__( self, root: str = "", @@ -60,13 +68,13 @@ def download(self) -> None: self._save_raw_data() def _get_trips(self) -> Iterator[TripletLike]: - return chain.from_iterable((iter(ds["graph"]) for ds in self.raw_dataset)) + return chain.from_iterable( + (iter(ds["graph"]) for ds in self.raw_dataset)) def _build_graph(self) -> None: trips = self._get_trips() self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( - trips, pre_transform=preprocess_triplet - ) + trips, pre_transform=preprocess_triplet) # Nodes: nodes = self.indexer.get_unique_node_features() @@ -74,7 +82,8 @@ def _build_graph(self) -> None: self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: - edges = self.indexer.get_unique_edge_features(feature_name=EDGE_RELATION) + edges = self.indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) edge_attr = self.model.encode(edges, batch_size=256) self.indexer.add_edge_feature( new_feature_name="edge_attr", @@ -97,15 +106,16 @@ def _retrieve_subgraphs(self) -> None: data_i = self.raw_dataset[index] local_trips = data_i["graph"] if self.whole_graph_retrieval: - graph = self.indexer.to_data( - node_feature_name="x", edge_feature_name="edge_attr" - ) + graph = self.indexer.to_data(node_feature_name="x", + edge_feature_name="edge_attr") else: graph = get_features_for_triplets( - self.indexer, local_trips, pre_transform=preprocess_triplet - ) - textual_nodes = self.textual_nodes.iloc[graph["node_idx"]].reset_index() - textual_edges = self.textual_edges.iloc[graph["edge_idx"]].reset_index() + self.indexer, local_trips, + pre_transform=preprocess_triplet) + textual_nodes = self.textual_nodes.iloc[ + graph["node_idx"]].reset_index() + textual_edges = self.textual_edges.iloc[ + graph["edge_idx"]].reset_index() self.raw_graphs.append(graph) pcst_subgraph, desc = retrieval_via_pcst( graph, @@ -137,13 +147,11 @@ def process(self) -> None: print("Loading graph...") self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) self.textual_nodes = DataFrame.from_dict( - {"node_attr": self.indexer.get_node_features()} - ) + {"node_attr": self.indexer.get_node_features()}) self.textual_nodes["node_id"] = self.textual_nodes.index self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] - self.textual_edges = DataFrame( - self.indexer.get_edge_features(), columns=["src", "edge_attr", "dst"] - ) + self.textual_edges = DataFrame(self.indexer.get_edge_features(), + columns=["src", "edge_attr", "dst"]) self.textual_edges["src"] = [ self.indexer._nodes[h] for h in self.textual_edges["src"] ] diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index a763026f6b53..584163dd4e6a 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -62,9 +62,8 @@ def __eq__(self, value: "MappedFeature") -> bool: class LargeGraphIndexer: """For a dataset that consists of mulitiple subgraphs that are assumed to be part of a much larger graph, collate the values into a large graph store - to save resources + to save resources. """ - def __init__( self, nodes: Iterable[Hashable], @@ -88,11 +87,11 @@ def __init__( self.node_attr = node_attr if NODE_KEYS & set(self.node_attr.keys()) != NODE_KEYS: raise AttributeError( - "Invalid node_attr object. Missing " - + f"{NODE_KEYS - set(self.node_attr.keys())}" - ) + "Invalid node_attr object. Missing " + + f"{NODE_KEYS - set(self.node_attr.keys())}") elif self.node_attr[NODE_PID] != nodes: - raise AttributeError("Nodes provided do not match those in node_attr") + raise AttributeError( + "Nodes provided do not match those in node_attr") else: self.node_attr = dict() self.node_attr[NODE_PID] = nodes @@ -106,11 +105,11 @@ def __init__( if EDGE_KEYS & set(self.edge_attr.keys()) != EDGE_KEYS: raise AttributeError( - "Invalid edge_attr object. Missing " - + f"{EDGE_KEYS - set(self.edge_attr.keys())}" - ) + "Invalid edge_attr object. Missing " + + f"{EDGE_KEYS - set(self.edge_attr.keys())}") elif self.node_attr[EDGE_PID] != edges: - raise AttributeError("Edges provided do not match those in edge_attr") + raise AttributeError( + "Edges provided do not match those in edge_attr") else: self.edge_attr = dict() @@ -123,7 +122,8 @@ def __init__( self.edge_attr[EDGE_HEAD].append(h) self.edge_attr[EDGE_RELATION].append(r) self.edge_attr[EDGE_TAIL].append(t) - self.edge_attr[EDGE_INDEX].append((self._nodes[h], self._nodes[t])) + self.edge_attr[EDGE_INDEX].append( + (self._nodes[h], self._nodes[t])) for i, tup in enumerate(edges): self._edges[tup] = i @@ -141,8 +141,7 @@ def from_triplets( if pre_transform is not None: def apply_transform( - trips: Iterable[TripletLike], - ) -> Iterator[TripletLike]: + trips: Iterable[TripletLike]) -> Iterator[TripletLike]: for trip in trips: yield pre_transform(trip) @@ -159,19 +158,23 @@ def apply_transform( return cls(list(nodes), list(edges)) @classmethod - def collate(cls, graphs: Iterable["LargeGraphIndexer"]) -> "LargeGraphIndexer": + def collate(cls, + graphs: Iterable["LargeGraphIndexer"]) -> "LargeGraphIndexer": # FIXME Needs to merge node attrs and edge attrs? trips = chain.from_iterable([graph.to_triplets() for graph in graphs]) return cls.from_triplets(trips) - def get_unique_node_features(self, feature_name: str = NODE_PID) -> List[Hashable]: + def get_unique_node_features( + self, feature_name: str = NODE_PID) -> List[Hashable]: try: if feature_name in self._mapped_node_features: - raise IndexError("Only non-mapped features can be retrieved uniquely.") + raise IndexError( + "Only non-mapped features can be retrieved uniquely.") return ordered_set(self.get_node_features(feature_name)) except KeyError: - raise AttributeError(f"Nodes do not have a feature called {feature_name}") + raise AttributeError( + f"Nodes do not have a feature called {feature_name}") def add_node_feature( self, @@ -183,21 +186,20 @@ def add_node_feature( if new_feature_name in self.node_attr: raise AttributeError("Features cannot be overridden once created") if map_from_feature in self._mapped_node_features: - raise AttributeError(f"{map_from_feature} is already a feature mapping.") + raise AttributeError( + f"{map_from_feature} is already a feature mapping.") feature_keys = self.get_unique_node_features(map_from_feature) if len(feature_keys) != len(new_feature_vals): raise AttributeError( - "Expected encodings for {len(feature_keys)} unique features," - + f" but got {len(new_feature_vals)} encodings." - ) + "Expected encodings for {len(feature_keys)} unique features," + + f" but got {len(new_feature_vals)} encodings.") if map_from_feature == NODE_PID: self.node_attr[new_feature_name] = new_feature_vals else: self.node_attr[new_feature_name] = MappedFeature( - name=map_from_feature, values=new_feature_vals - ) + name=map_from_feature, values=new_feature_vals) self._mapped_node_features.add(new_feature_name) def get_node_features( @@ -211,8 +213,8 @@ def get_node_features( values = self.node_attr[feature_name] if isinstance(values, torch.Tensor): idxs = list( - self.get_node_features_iter(feature_name, pids, index_only=True) - ) + self.get_node_features_iter(feature_name, pids, + index_only=True)) return values[idxs] return list(self.get_node_features_iter(feature_name, pids)) @@ -231,7 +233,8 @@ def get_node_features_iter( feature_map_info.name, feature_map_info.values, ) - from_feature_vals = self.get_unique_node_features(from_feature_name) + from_feature_vals = self.get_unique_node_features( + from_feature_name) feature_mapping = {k: i for i, k in enumerate(from_feature_vals)} for pid in pids: @@ -250,13 +253,16 @@ def get_node_features_iter( else: yield self.node_attr[feature_name][idx] - def get_unique_edge_features(self, feature_name: str = EDGE_PID) -> List[Hashable]: + def get_unique_edge_features( + self, feature_name: str = EDGE_PID) -> List[Hashable]: try: if feature_name in self._mapped_edge_features: - raise IndexError("Only non-mapped features can be retrieved uniquely.") + raise IndexError( + "Only non-mapped features can be retrieved uniquely.") return ordered_set(self.get_edge_features(feature_name)) except KeyError: - raise AttributeError(f"Edges do not have a feature called {feature_name}") + raise AttributeError( + f"Edges do not have a feature called {feature_name}") def add_edge_feature( self, @@ -268,21 +274,20 @@ def add_edge_feature( if new_feature_name in self.edge_attr: raise AttributeError("Features cannot be overridden once created") if map_from_feature in self._mapped_edge_features: - raise AttributeError(f"{map_from_feature} is already a feature mapping.") + raise AttributeError( + f"{map_from_feature} is already a feature mapping.") feature_keys = self.get_unique_edge_features(map_from_feature) if len(feature_keys) != len(new_feature_vals): raise AttributeError( "Expected encodings for {len(feature_keys)} unique features, " - + f"but got {len(new_feature_vals)} encodings." - ) + + f"but got {len(new_feature_vals)} encodings.") if map_from_feature == EDGE_PID: self.edge_attr[new_feature_name] = new_feature_vals else: self.edge_attr[new_feature_name] = MappedFeature( - name=map_from_feature, values=new_feature_vals - ) + name=map_from_feature, values=new_feature_vals) self._mapped_edge_features.add(new_feature_name) def get_edge_features( @@ -296,8 +301,8 @@ def get_edge_features( values = self.edge_attr[feature_name] if isinstance(values, torch.Tensor): idxs = list( - self.get_edge_features_iter(feature_name, pids, index_only=True) - ) + self.get_edge_features_iter(feature_name, pids, + index_only=True)) return values[idxs] return list(self.get_edge_features_iter(feature_name, pids)) @@ -316,7 +321,8 @@ def get_edge_features_iter( feature_map_info.name, feature_map_info.values, ) - from_feature_vals = self.get_unique_edge_features(from_feature_name) + from_feature_vals = self.get_unique_edge_features( + from_feature_name) feature_mapping = {k: i for i, k in enumerate(from_feature_vals)} for pid in pids: @@ -416,20 +422,16 @@ def _to_data_attrs( self, node_feature_name: str, edge_feature_name: Optional[str] = None ) -> Tuple[torch.Tensor, torch.LongTensor, Optional[torch.Tensor]]: x = torch.Tensor(self.get_node_features(node_feature_name)) - edge_index = torch.t(torch.LongTensor(self.get_edge_features(EDGE_INDEX))) - edge_attr = ( - self.get_edge_features(edge_feature_name) - if edge_feature_name is not None - else None - ) + edge_index = torch.t( + torch.LongTensor(self.get_edge_features(EDGE_INDEX))) + edge_attr = (self.get_edge_features(edge_feature_name) + if edge_feature_name is not None else None) return x, edge_index, edge_attr - def to_data( - self, node_feature_name: str, edge_feature_name: Optional[str] = None - ) -> Data: - x, edge_index, edge_attr = self._to_data_attrs( - node_feature_name, edge_feature_name - ) + def to_data(self, node_feature_name: str, + edge_feature_name: Optional[str] = None) -> Data: + x, edge_index, edge_attr = self._to_data_attrs(node_feature_name, + edge_feature_name) return Data(x=x, edge_index=edge_index, edge_attr=edge_attr) @@ -451,17 +453,14 @@ def apply_transform(trips): triplets = list(apply_transform(triplets)) small_graph_indexer = LargeGraphIndexer.from_triplets( - triplets, pre_transform=pre_transform - ) + triplets, pre_transform=pre_transform) node_keys = small_graph_indexer.get_node_features() - node_feats = indexer.get_node_features( - feature_name=node_feature_name, pids=node_keys - ) + node_feats = indexer.get_node_features(feature_name=node_feature_name, + pids=node_keys) edge_keys = small_graph_indexer.get_edge_features(pids=triplets) - edge_feats = indexer.get_edge_features( - feature_name=edge_feature_name, pids=edge_keys - ) + edge_feats = indexer.get_edge_features(feature_name=edge_feature_name, + pids=edge_keys) edge_index = small_graph_indexer.get_edge_features(EDGE_INDEX, triplets) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 81b6b782f348..5271ab50be1c 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -43,11 +43,9 @@ def retrieval_via_pcst( ) -> Tuple[Data, str]: c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = ( - textual_nodes.to_csv(index=False) - + "\n" - + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"]) - ) + desc = (textual_nodes.to_csv(index=False) + + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"])) graph = Data( x=graph.x, edge_index=graph.edge_index, @@ -115,9 +113,8 @@ def retrieval_via_pcst( costs = np.array(costs + virtual_costs) edges = np.array(edges + virtual_edges) - vertices, edges = pcst_fast( - edges, prizes, costs, root, num_clusters, pruning, verbosity_level - ) + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) selected_nodes = vertices[vertices < graph.num_nodes] selected_edges = [mapping_e[e] for e in edges if e < num_edges] @@ -129,16 +126,13 @@ def retrieval_via_pcst( edge_index = graph.edge_index[:, selected_edges] selected_nodes = np.unique( - np.concatenate([selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()]) - ) + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = ( - n.to_csv(index=False) - + "\n" - + e.to_csv(index=False, columns=["src", "edge_attr", "dst"]) - ) + desc = (n.to_csv(index=False) + "\n" + + e.to_csv(index=False, columns=["src", "edge_attr", "dst"])) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -147,9 +141,8 @@ def retrieval_via_pcst( src = [mapping[i] for i in edge_index[0].tolist()] dst = [mapping[i] for i in edge_index[1].tolist()] edge_index = torch.LongTensor([src, dst]) - data = Data( - x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes) - ) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) return data, desc @@ -170,19 +163,18 @@ class WebQSPDataset(InMemoryDataset): force_reload (bool, optional): Whether to re-process the dataset. (default: :obj:`False`) """ - def __init__( self, root: str = "", force_reload: bool = False, ) -> None: self._check_dependencies() - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) self.load(self.processed_paths[0]) def _check_dependencies(self) -> None: - missing_imports = False missing_str_list = [] if not WITH_PCST: missing_str_list.append('pcst_fast') @@ -206,14 +198,15 @@ def processed_file_names(self) -> List[str]: def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( - [dataset["train"], dataset["validation"], dataset["test"]] - ) + [dataset["train"], dataset["validation"], dataset["test"]]) self.split_idxs = { - "train": torch.arange(len(dataset["train"])), - "val": torch.arange(len(dataset["validation"])) + len(dataset["train"]), - "test": torch.arange(len(dataset["test"])) - + len(dataset["train"]) - + len(dataset["validation"]), + "train": + torch.arange(len(dataset["train"])), + "val": + torch.arange(len(dataset["validation"])) + len(dataset["train"]), + "test": + torch.arange(len(dataset["test"])) + len(dataset["train"]) + + len(dataset["validation"]), } def process(self) -> None: From ae8d0571ba359ecefb5eca4270f981aece00b3a5 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 18:12:28 -0700 Subject: [PATCH 599/752] start unittests --- test/data/test_large_graph_indexer.py | 89 +++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/data/test_large_graph_indexer.py diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py new file mode 100644 index 000000000000..e91b483c0898 --- /dev/null +++ b/test/data/test_large_graph_indexer.py @@ -0,0 +1,89 @@ +import pytest +import random +import string +from torch_geometric.data import LargeGraphIndexer, TripletLike, Data, get_features_for_triplets +from torch_geometric.data.large_graph_indexer import NODE_PID, EDGE_RELATION +from typing import List +import torch + +# create possible nodes and edges for graph +strkeys = string.ascii_letters + string.digits +NODE_POOL = set([random.sample(strkeys, 10) for i in range(1000)]) +EDGE_POOL = set([random.sample(strkeys, 10) for i in range(50)]) + +def featurize(s: str) -> int: + return int.from_bytes(s.encode(), 'little') + +def sample_triplets(amount: int = 1) -> List[TripletLike]: + trips = [] + for i in range(amount): + trips.append(random.choice(NODE_POOL), random.choice(EDGE_POOL), random.choice(NODE_POOL)) + return trips + +def preprocess_triplet(triplet: TripletLike) -> TripletLike: + h, r, t = triplet + return h.lower(), r, t.lower() + + +def test_basic_collate(): + graphs = [sample_triplets(1000) for i in range(2)] + + indexer_0 = LargeGraphIndexer.from_triplets(graphs[0], pre_transform=preprocess_triplet) + indexer_1 = LargeGraphIndexer.from_triplets(graphs[1], pre_transform=preprocess_triplet) + + big_indexer = LargeGraphIndexer.collate([indexer_0, indexer_1]) + + assert len(indexer_0._nodes) + len(indexer_1._nodes) - len(indexer_0._nodes.keys() & indexer_1._nodes.keys()) == len(big_indexer._nodes) + assert len(indexer_0._edges) + len(indexer_1._edges) - len(indexer_0._edges.keys() & indexer_1._edges.keys()) == len(big_indexer._edges) + + assert len(set(big_indexer._nodes.values())) == len(big_indexer._nodes) + assert len(set(big_indexer._edges.values())) == len(big_indexer._edges) + + for node in (indexer_0._nodes.keys() | indexer_1._nodes.keys()): + assert big_indexer.node_attr[NODE_PID][big_indexer._nodes[node]] == node + + +def test_large_graph_index(): + graphs = [sample_triplets(1000) for i in range(100)] + + node_feature_vecs = [featurize(s) for s in NODE_POOL] + edge_feature_vecs = [featurize(s) for s in EDGE_POOL] + + def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: + seen_nodes = dict() + edge_attrs = list() + edge_idx = [] + for trip in triplets: + h, r, t = trip + seen_nodes[h] = len(seen_nodes) if h not in seen_nodes else seen_nodes[h] + seen_nodes[t] = len(seen_nodes) if t not in seen_nodes else seen_nodes[t] + edge_attrs.append(edge_feature_vecs[r]) + edge_idx.append((seen_nodes[h], seen_nodes[t])) + + x = torch.Tensor([node_feature_vecs[n] for n in seen_nodes.keys()]) + edge_idx = torch.LongTensor(edge_idx).T + edge_attrs = torch.Tensor(edge_attrs) + return Data(x=x, edge_index=edge_idx, edge_attr=edge_attrs) + + naive_graph_ds = [encode_graph_from_trips(triplets=trips) for trips in graphs] + + indexer = LargeGraphIndexer.collate([LargeGraphIndexer.from_triplets(g) for g in graphs]) + indexer.add_node_feature('x', torch.Tensor(node_feature_vecs)) + indexer.add_edge_feature('edge_attr', torch.Tensor(edge_feature_vecs), map_from_feature=EDGE_RELATION) + large_graph_ds = [get_features_for_triplets(indexer=indexer, triplets=g, node_feature_name='x', edge_feature_name='edge_attr') for g in graphs] + assert naive_graph_ds == large_graph_ds + +def test_save_load(tmp_path): + graphs = [sample_triplets(1000)] + + node_feature_vecs = [featurize(s) for s in NODE_POOL] + edge_feature_vecs = [featurize(s) for s in EDGE_POOL] + + indexer = LargeGraphIndexer.from_triplets(graphs) + indexer.add_node_feature('x', torch.Tensor(node_feature_vecs)) + indexer.add_edge_feature('edge_attr', torch.Tensor(edge_feature_vecs), map_from_feature=EDGE_RELATION) + + indexer.save(tmp_path) + assert indexer == LargeGraphIndexer.from_disk(tmp_path) + + From 7fdae854a43fffd3677cf55dff4033b005ab0410 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 18:51:25 -0700 Subject: [PATCH 600/752] tests done for largegraphindexer --- test/data/test_large_graph_indexer.py | 139 +++++++++++++++++++------- 1 file changed, 103 insertions(+), 36 deletions(-) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index e91b483c0898..a57c59535258 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -1,25 +1,38 @@ -import pytest import random import string -from torch_geometric.data import LargeGraphIndexer, TripletLike, Data, get_features_for_triplets -from torch_geometric.data.large_graph_indexer import NODE_PID, EDGE_RELATION from typing import List + +import networkx as nx import torch +from torch_geometric.data import ( + Data, + LargeGraphIndexer, + TripletLike, + get_features_for_triplets, +) +from torch_geometric.data.large_graph_indexer import EDGE_RELATION, NODE_PID + # create possible nodes and edges for graph strkeys = string.ascii_letters + string.digits -NODE_POOL = set([random.sample(strkeys, 10) for i in range(1000)]) -EDGE_POOL = set([random.sample(strkeys, 10) for i in range(50)]) +NODE_POOL = list( + set(["".join(random.sample(strkeys, 10)) for i in range(1000)])) +EDGE_POOL = list(set(["".join(random.sample(strkeys, 10)) for i in range(50)])) + def featurize(s: str) -> int: return int.from_bytes(s.encode(), 'little') + def sample_triplets(amount: int = 1) -> List[TripletLike]: trips = [] for i in range(amount): - trips.append(random.choice(NODE_POOL), random.choice(EDGE_POOL), random.choice(NODE_POOL)) + h, t = random.sample(NODE_POOL, k=2) + r = random.sample(EDGE_POOL, k=1)[0] + trips.append(tuple([h, r, t])) return trips + def preprocess_triplet(triplet: TripletLike) -> TripletLike: h, r, t = triplet return h.lower(), r, t.lower() @@ -28,26 +41,35 @@ def preprocess_triplet(triplet: TripletLike) -> TripletLike: def test_basic_collate(): graphs = [sample_triplets(1000) for i in range(2)] - indexer_0 = LargeGraphIndexer.from_triplets(graphs[0], pre_transform=preprocess_triplet) - indexer_1 = LargeGraphIndexer.from_triplets(graphs[1], pre_transform=preprocess_triplet) + indexer_0 = LargeGraphIndexer.from_triplets( + graphs[0], pre_transform=preprocess_triplet) + indexer_1 = LargeGraphIndexer.from_triplets( + graphs[1], pre_transform=preprocess_triplet) big_indexer = LargeGraphIndexer.collate([indexer_0, indexer_1]) - assert len(indexer_0._nodes) + len(indexer_1._nodes) - len(indexer_0._nodes.keys() & indexer_1._nodes.keys()) == len(big_indexer._nodes) - assert len(indexer_0._edges) + len(indexer_1._edges) - len(indexer_0._edges.keys() & indexer_1._edges.keys()) == len(big_indexer._edges) + assert len(indexer_0._nodes) + len( + indexer_1._nodes) - len(indexer_0._nodes.keys() + & indexer_1._nodes.keys()) == len( + big_indexer._nodes) + assert len(indexer_0._edges) + len( + indexer_1._edges) - len(indexer_0._edges.keys() + & indexer_1._edges.keys()) == len( + big_indexer._edges) assert len(set(big_indexer._nodes.values())) == len(big_indexer._nodes) assert len(set(big_indexer._edges.values())) == len(big_indexer._edges) for node in (indexer_0._nodes.keys() | indexer_1._nodes.keys()): - assert big_indexer.node_attr[NODE_PID][big_indexer._nodes[node]] == node + assert big_indexer.node_attr[NODE_PID][ + big_indexer._nodes[node]] == node def test_large_graph_index(): graphs = [sample_triplets(1000) for i in range(100)] - node_feature_vecs = [featurize(s) for s in NODE_POOL] - edge_feature_vecs = [featurize(s) for s in EDGE_POOL] + node_feature_vecs = {s: featurize(s) for s in NODE_POOL} + edge_feature_vecs = {s: featurize(s) for s in EDGE_POOL} def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: seen_nodes = dict() @@ -55,35 +77,80 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: edge_idx = [] for trip in triplets: h, r, t = trip - seen_nodes[h] = len(seen_nodes) if h not in seen_nodes else seen_nodes[h] - seen_nodes[t] = len(seen_nodes) if t not in seen_nodes else seen_nodes[t] + seen_nodes[h] = len( + seen_nodes) if h not in seen_nodes else seen_nodes[h] + seen_nodes[t] = len( + seen_nodes) if t not in seen_nodes else seen_nodes[t] edge_attrs.append(edge_feature_vecs[r]) edge_idx.append((seen_nodes[h], seen_nodes[t])) - + x = torch.Tensor([node_feature_vecs[n] for n in seen_nodes.keys()]) edge_idx = torch.LongTensor(edge_idx).T edge_attrs = torch.Tensor(edge_attrs) return Data(x=x, edge_index=edge_idx, edge_attr=edge_attrs) - - naive_graph_ds = [encode_graph_from_trips(triplets=trips) for trips in graphs] - indexer = LargeGraphIndexer.collate([LargeGraphIndexer.from_triplets(g) for g in graphs]) - indexer.add_node_feature('x', torch.Tensor(node_feature_vecs)) - indexer.add_edge_feature('edge_attr', torch.Tensor(edge_feature_vecs), map_from_feature=EDGE_RELATION) - large_graph_ds = [get_features_for_triplets(indexer=indexer, triplets=g, node_feature_name='x', edge_feature_name='edge_attr') for g in graphs] - assert naive_graph_ds == large_graph_ds - -def test_save_load(tmp_path): - graphs = [sample_triplets(1000)] - - node_feature_vecs = [featurize(s) for s in NODE_POOL] - edge_feature_vecs = [featurize(s) for s in EDGE_POOL] - - indexer = LargeGraphIndexer.from_triplets(graphs) - indexer.add_node_feature('x', torch.Tensor(node_feature_vecs)) - indexer.add_edge_feature('edge_attr', torch.Tensor(edge_feature_vecs), map_from_feature=EDGE_RELATION) - - indexer.save(tmp_path) - assert indexer == LargeGraphIndexer.from_disk(tmp_path) + naive_graph_ds = [ + encode_graph_from_trips(triplets=trips) for trips in graphs + ] + + indexer = LargeGraphIndexer.collate( + [LargeGraphIndexer.from_triplets(g) for g in graphs]) + indexer_nodes = indexer.get_unique_node_features() + indexer_node_vals = torch.Tensor( + [node_feature_vecs[n] for n in indexer_nodes]) + indexer_edges = indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) + indexer_edge_vals = torch.Tensor( + [edge_feature_vecs[e] for e in indexer_edges]) + indexer.add_node_feature('x', indexer_node_vals) + indexer.add_edge_feature('edge_attr', indexer_edge_vals, + map_from_feature=EDGE_RELATION) + large_graph_ds = [ + get_features_for_triplets(indexer=indexer, triplets=g, + node_feature_name='x', + edge_feature_name='edge_attr') + for g in graphs + ] + + def results_are_close_enough(ground_truth: Data, new_method: Data, + thresh=.9): + def _sorted_tensors_are_close(tensor1, tensor2): + return torch.all( + torch.isclose(tensor1.sort()[0], + tensor2.sort()[0]) > thresh) + + def _graphs_are_same(tensor1, tensor2): + return nx.weisfeiler_lehman_graph_hash(nx.Graph( + tensor1.T)) == nx.weisfeiler_lehman_graph_hash( + nx.Graph(tensor2.T)) + return _sorted_tensors_are_close( + ground_truth.x, new_method.x) \ + and _sorted_tensors_are_close( + ground_truth.edge_attr, new_method.edge_attr) \ + and _graphs_are_same( + ground_truth.edge_index, new_method.edge_index) + + for dsets in zip(naive_graph_ds, large_graph_ds): + assert results_are_close_enough(*dsets) +def test_save_load(tmp_path): + graph = sample_triplets(1000) + + node_feature_vecs = {s: featurize(s) for s in NODE_POOL} + edge_feature_vecs = {s: featurize(s) for s in EDGE_POOL} + + indexer = LargeGraphIndexer.from_triplets(graph) + indexer_nodes = indexer.get_unique_node_features() + indexer_node_vals = torch.Tensor( + [node_feature_vecs[n] for n in indexer_nodes]) + indexer_edges = indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) + indexer_edge_vals = torch.Tensor( + [edge_feature_vecs[e] for e in indexer_edges]) + indexer.add_node_feature('x', indexer_node_vals) + indexer.add_edge_feature('edge_attr', indexer_edge_vals, + map_from_feature=EDGE_RELATION) + + indexer.save(str(tmp_path)) + assert indexer == LargeGraphIndexer.from_disk(str(tmp_path)) From 5f67a840fa1cb7b97c8b69494c67769b6aba7799 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 18:53:09 -0700 Subject: [PATCH 601/752] tests done for largegraphindexer 2 --- test/data/test_large_graph_indexer.py | 2 +- torch_geometric/data/__init__.py | 12 +++++------- torch_geometric/data/large_graph_indexer.py | 5 ----- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index a57c59535258..39e1e302ead9 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -113,7 +113,7 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: ] def results_are_close_enough(ground_truth: Data, new_method: Data, - thresh=.9): + thresh=.99): def _sorted_tensors_are_close(tensor1, tensor2): return torch.all( torch.isclose(tensor1.sort()[0], diff --git a/torch_geometric/data/__init__.py b/torch_geometric/data/__init__.py index 821ef9c5c063..0a61bea38315 100644 --- a/torch_geometric/data/__init__.py +++ b/torch_geometric/data/__init__.py @@ -16,6 +16,7 @@ from .makedirs import makedirs from .download import download_url, download_google_url from .extract import extract_tar, extract_zip, extract_bz2, extract_gz +from .large_graph_indexer import LargeGraphIndexer, TripletLike, get_features_for_triplets from torch_geometric.lazy_loader import LazyLoader @@ -27,6 +28,8 @@ 'Dataset', 'InMemoryDataset', 'OnDiskDataset', + 'LargeGraphIndexer', + 'TripletLike', ] remote_backend_classes = [ @@ -43,13 +46,8 @@ ] helper_functions = [ - 'makedirs', - 'download_url', - 'download_google_url', - 'extract_tar', - 'extract_zip', - 'extract_bz2', - 'extract_gz', + 'makedirs', 'download_url', 'download_google_url', 'extract_tar', + 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets' ] __all__ = data_classes + remote_backend_classes + helper_functions diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 584163dd4e6a..1104ed9ac1c5 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -473,10 +473,5 @@ def apply_transform(trips): edge_attr=edge_attr, num_nodes=len(node_feats), ) - # needed for mappings - data_obj[NODE_PID] = node_keys - data_obj[EDGE_PID] = edge_keys - data_obj["node_idx"] = [indexer._nodes[k] for k in node_keys] - data_obj["edge_idx"] = [indexer._edges[e] for e in edge_keys] return data_obj From b0ab66f0d7615d15309396a16323231aafd6327a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 18:59:35 -0700 Subject: [PATCH 602/752] tests done for largegraphindexer 3 --- test/data/test_large_graph_indexer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index 39e1e302ead9..89d7c48f370c 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -68,7 +68,8 @@ def test_basic_collate(): def test_large_graph_index(): graphs = [sample_triplets(1000) for i in range(100)] - node_feature_vecs = {s: featurize(s) for s in NODE_POOL} + # Preprocessing of trips lowercases nodes but not edges + node_feature_vecs = {s.lower(): featurize(s.lower()) for s in NODE_POOL} edge_feature_vecs = {s: featurize(s) for s in EDGE_POOL} def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: @@ -76,6 +77,7 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: edge_attrs = list() edge_idx = [] for trip in triplets: + trip = preprocess_triplet(trip) h, r, t = trip seen_nodes[h] = len( seen_nodes) if h not in seen_nodes else seen_nodes[h] @@ -94,7 +96,7 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: ] indexer = LargeGraphIndexer.collate( - [LargeGraphIndexer.from_triplets(g) for g in graphs]) + [LargeGraphIndexer.from_triplets(g, pre_transform=preprocess_triplet) for g in graphs]) indexer_nodes = indexer.get_unique_node_features() indexer_node_vals = torch.Tensor( [node_feature_vecs[n] for n in indexer_nodes]) @@ -108,7 +110,7 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: large_graph_ds = [ get_features_for_triplets(indexer=indexer, triplets=g, node_feature_name='x', - edge_feature_name='edge_attr') + edge_feature_name='edge_attr', pre_transform=preprocess_triplet) for g in graphs ] From bfd1a2f9e8ecdcb07cdeea03ddff641975f16841 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:00:20 -0700 Subject: [PATCH 603/752] tests done for largegraphindexer 4 --- test/data/test_large_graph_indexer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index 89d7c48f370c..f9a6eebf573c 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -95,8 +95,10 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: encode_graph_from_trips(triplets=trips) for trips in graphs ] - indexer = LargeGraphIndexer.collate( - [LargeGraphIndexer.from_triplets(g, pre_transform=preprocess_triplet) for g in graphs]) + indexer = LargeGraphIndexer.collate([ + LargeGraphIndexer.from_triplets(g, pre_transform=preprocess_triplet) + for g in graphs + ]) indexer_nodes = indexer.get_unique_node_features() indexer_node_vals = torch.Tensor( [node_feature_vecs[n] for n in indexer_nodes]) @@ -110,7 +112,8 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: large_graph_ds = [ get_features_for_triplets(indexer=indexer, triplets=g, node_feature_name='x', - edge_feature_name='edge_attr', pre_transform=preprocess_triplet) + edge_feature_name='edge_attr', + pre_transform=preprocess_triplet) for g in graphs ] From bcee67a530fd5367cf23b25441d0d6e636d6f1d1 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:13:56 -0700 Subject: [PATCH 604/752] migrate updated qsp dataset --- examples/llm_plus_gnn/updated_qsp_dataset.py | 161 --------- .../datasets/updated_web_qsp_dataset.py | 315 ++++++++++++++++++ 2 files changed, 315 insertions(+), 161 deletions(-) delete mode 100644 examples/llm_plus_gnn/updated_qsp_dataset.py create mode 100644 torch_geometric/datasets/updated_web_qsp_dataset.py diff --git a/examples/llm_plus_gnn/updated_qsp_dataset.py b/examples/llm_plus_gnn/updated_qsp_dataset.py deleted file mode 100644 index 97e519d64056..000000000000 --- a/examples/llm_plus_gnn/updated_qsp_dataset.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -from itertools import chain -from typing import Iterator, List - -import torch -from large_graph_indexer import ( - EDGE_RELATION, - LargeGraphIndexer, - TripletLike, - get_features_for_triplets, -) -from tqdm import tqdm - -from torch_geometric.datasets.web_qsp_dataset import ( - DataFrame, - WebQSPDataset, - datasets, - retrieval_via_pcst, -) -from torch_geometric.nn.nlp import SentenceTransformer - - -def preprocess_triplet(triplet: TripletLike) -> TripletLike: - h, r, t = triplet - return h.lower(), r, t.lower() - - -class UpdatedWebQSPDataset(WebQSPDataset): - def __init__( - self, - root: str = "", - force_reload: bool = False, - whole_graph_retrieval: bool = False, - limit: int = -1, - ) -> None: - self.limit = limit - self.whole_graph_retrieval = whole_graph_retrieval - super().__init__(root, force_reload) - - @property - def raw_file_names(self) -> List[str]: - return ["raw_data", "split_idxs"] - - @property - def processed_file_names(self) -> List[str]: - return [ - "list_of_graphs.pt", - "pre_filter.pt", - "pre_transform.pt", - "raw_graphs.pt", - "large_graph_indexer", - ] - - def _save_raw_data(self) -> None: - self.raw_dataset.save_to_disk(self.raw_paths[0]) - torch.save(self.split_idxs, self.raw_paths[1]) - - def _load_raw_data(self) -> None: - if not hasattr(self, "raw_dataset"): - self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) - if not hasattr(self, "split_idxs"): - self.split_idxs = torch.load(self.raw_paths[1]) - - def download(self) -> None: - super().download() - if self.limit >= 0: - self.raw_dataset = self.raw_dataset.select(range(self.limit)) - self._save_raw_data() - - def _get_trips(self) -> Iterator[TripletLike]: - return chain.from_iterable( - (iter(ds["graph"]) for ds in self.raw_dataset)) - - def _build_graph(self) -> None: - trips = self._get_trips() - self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( - trips, pre_transform=preprocess_triplet) - - # Nodes: - nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256) - self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) - - # Edges: - edges = self.indexer.get_unique_edge_features( - feature_name=EDGE_RELATION) - edge_attr = self.model.encode(edges, batch_size=256) - self.indexer.add_edge_feature( - new_feature_name="edge_attr", - new_feature_vals=edge_attr, - map_from_feature=EDGE_RELATION, - ) - - self.indexer.save(self.processed_paths[-1]) - - def _retrieve_subgraphs(self) -> None: - print("Encoding questions...") - self.questions = [ds["question"] for ds in self.raw_dataset] - q_embs = self.model.encode(self.questions, batch_size=256) - list_of_graphs = [] - self.raw_graphs = [] - print("Retrieving subgraphs...") - textual_nodes = self.textual_nodes - textual_edges = self.textual_edges - for index in tqdm(range(len(self.raw_dataset))): - data_i = self.raw_dataset[index] - local_trips = data_i["graph"] - if self.whole_graph_retrieval: - graph = self.indexer.to_data(node_feature_name="x", - edge_feature_name="edge_attr") - else: - graph = get_features_for_triplets( - self.indexer, local_trips, - pre_transform=preprocess_triplet) - textual_nodes = self.textual_nodes.iloc[ - graph["node_idx"]].reset_index() - textual_edges = self.textual_edges.iloc[ - graph["edge_idx"]].reset_index() - self.raw_graphs.append(graph) - pcst_subgraph, desc = retrieval_via_pcst( - graph, - q_embs[index], - textual_nodes, - textual_edges, - topk=3, - topk_e=5, - cost_e=0.5, - ) - question = f"Question: {data_i['question']}\nAnswer: " - label = ("|").join(data_i["answer"]).lower() - - pcst_subgraph["question"] = question - pcst_subgraph["label"] = label - pcst_subgraph["desc"] = desc - list_of_graphs.append(pcst_subgraph.to("cpu")) - torch.save(self.raw_graphs, self.processed_paths[-2]) - self.save(list_of_graphs, self.processed_paths[0]) - - def process(self) -> None: - self._load_raw_data() - self.model = SentenceTransformer().to(self.device) - self.model.eval() - if not os.path.exists(self.processed_dir[-1]): - print("Encoding graph...") - self._build_graph() - else: - print("Loading graph...") - self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) - self.textual_nodes = DataFrame.from_dict( - {"node_attr": self.indexer.get_node_features()}) - self.textual_nodes["node_id"] = self.textual_nodes.index - self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] - self.textual_edges = DataFrame(self.indexer.get_edge_features(), - columns=["src", "edge_attr", "dst"]) - self.textual_edges["src"] = [ - self.indexer._nodes[h] for h in self.textual_edges["src"] - ] - self.textual_edges["dst"] = [ - self.indexer._nodes[h] for h in self.textual_edges["dst"] - ] - self._retrieve_subgraphs() diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py new file mode 100644 index 000000000000..1ec9c97b7347 --- /dev/null +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -0,0 +1,315 @@ +import os +from itertools import chain +from typing import Iterator, List, no_type_check, Tuple + +import numpy as np +import torch +from tqdm import tqdm + +from torch_geometric.data import InMemoryDataset, LargeGraphIndexer, TripletLike, get_features_for_triplets, Data +from torch_geometric.data.large_graph_indexer import EDGE_RELATION +from torch_geometric.nn.nlp import SentenceTransformer + +try: + from pandas import DataFrame + WITH_PANDAS = True +except ImportError: + DataFrame = None + WITH_PANDAS = False + +try: + from pcst_fast import pcst_fast + + WITH_PCST = True +except ImportError: + WITH_PCST = False + +try: + import datasets + + WITH_DATASETS = True +except ImportError: + WITH_DATASETS = False + +@no_type_check +def retrieval_via_pcst( + graph: Data, + q_emb: torch.Tensor, + textual_nodes: DataFrame, + textual_edges: DataFrame, + topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5, +) -> Tuple[Data, str]: + c = 0.01 + if len(textual_nodes) == 0 or len(textual_edges) == 0: + desc = (textual_nodes.to_csv(index=False) + + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"])) + graph = Data( + x=graph.x, + edge_index=graph.edge_index, + edge_attr=graph.edge_attr, + num_nodes=graph.num_nodes, + ) + return graph, desc + + root = -1 # unrooted + num_clusters = 1 + pruning = "gw" + verbosity_level = 0 + if topk > 0: + n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) + topk = min(topk, graph.num_nodes) + _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) + + n_prizes = torch.zeros_like(n_prizes) + n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() + else: + n_prizes = torch.zeros(graph.num_nodes) + + if topk_e > 0: + e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) + topk_e = min(topk_e, e_prizes.unique().size(0)) + + topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) + e_prizes[e_prizes < topk_e_values[-1]] = 0.0 + last_topk_e_value = topk_e + for k in range(topk_e): + indices = e_prizes == topk_e_values[k] + value = min((topk_e - k) / sum(indices), last_topk_e_value - c) + e_prizes[indices] = value + last_topk_e_value = value + # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) + else: + e_prizes = torch.zeros(graph.num_edges) + + costs = [] + edges = [] + virtual_n_prizes = [] + virtual_edges = [] + virtual_costs = [] + mapping_n = {} + mapping_e = {} + for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): + prize_e = e_prizes[i] + if prize_e <= cost_e: + mapping_e[len(edges)] = i + edges.append((src, dst)) + costs.append(cost_e - prize_e) + else: + virtual_node_id = graph.num_nodes + len(virtual_n_prizes) + mapping_n[virtual_node_id] = i + virtual_edges.append((src, virtual_node_id)) + virtual_edges.append((virtual_node_id, dst)) + virtual_costs.append(0) + virtual_costs.append(0) + virtual_n_prizes.append(prize_e - cost_e) + + prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) + num_edges = len(edges) + if len(virtual_costs) > 0: + costs = np.array(costs + virtual_costs) + edges = np.array(edges + virtual_edges) + + vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, + pruning, verbosity_level) + + selected_nodes = vertices[vertices < graph.num_nodes] + selected_edges = [mapping_e[e] for e in edges if e < num_edges] + virtual_vertices = vertices[vertices >= graph.num_nodes] + if len(virtual_vertices) > 0: + virtual_vertices = vertices[vertices >= graph.num_nodes] + virtual_edges = [mapping_n[i] for i in virtual_vertices] + selected_edges = np.array(selected_edges + virtual_edges) + + edge_index = graph.edge_index[:, selected_edges] + selected_nodes = np.unique( + np.concatenate( + [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) + + n = textual_nodes.iloc[selected_nodes] + e = textual_edges.iloc[selected_edges] + desc = (n.to_csv(index=False) + "\n" + + e.to_csv(index=False, columns=["src", "edge_attr", "dst"])) + + mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} + + x = graph.x[selected_nodes] + edge_attr = graph.edge_attr[selected_edges] + src = [mapping[i] for i in edge_index[0].tolist()] + dst = [mapping[i] for i in edge_index[1].tolist()] + edge_index = torch.LongTensor([src, dst]) + data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + num_nodes=len(selected_nodes)) + + return data, desc + + +def preprocess_triplet(triplet: TripletLike) -> TripletLike: + h, r, t = triplet + return h.lower(), r, t.lower() + + +class UpdatedWebQSPDataset(InMemoryDataset): + def __init__( + self, + root: str = "", + force_reload: bool = False, + whole_graph_retrieval: bool = False, + limit: int = -1, + ) -> None: + self.limit = limit + self.whole_graph_retrieval = whole_graph_retrieval + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + super().__init__(root, None, None, force_reload=force_reload) + self.load(self.processed_paths[0]) + + def _check_dependencies(self) -> None: + missing_str_list = [] + if not WITH_PCST: + missing_str_list.append('pcst_fast') + if not WITH_DATASETS: + missing_str_list.append('datasets') + if not WITH_PANDAS: + missing_str_list.append('pandas') + if len(missing_str_list) > 0: + missing_str = ' '.join(missing_str_list) + error_out = f"`pip install {missing_str}` to use this dataset." + raise ImportError(error_out) + + @property + def raw_file_names(self) -> List[str]: + return ["raw_data", "split_idxs"] + + @property + def processed_file_names(self) -> List[str]: + return [ + "list_of_graphs.pt", + "pre_filter.pt", + "pre_transform.pt", + "raw_graphs.pt", + "large_graph_indexer", + ] + + def _save_raw_data(self) -> None: + self.raw_dataset.save_to_disk(self.raw_paths[0]) + torch.save(self.split_idxs, self.raw_paths[1]) + + def _load_raw_data(self) -> None: + if not hasattr(self, "raw_dataset"): + self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) + if not hasattr(self, "split_idxs"): + self.split_idxs = torch.load(self.raw_paths[1]) + + def download(self) -> None: + dataset = datasets.load_dataset("rmanluo/RoG-webqsp") + self.raw_dataset = datasets.concatenate_datasets( + [dataset["train"], dataset["validation"], dataset["test"]]) + self.split_idxs = { + "train": + torch.arange(len(dataset["train"])), + "val": + torch.arange(len(dataset["validation"])) + len(dataset["train"]), + "test": + torch.arange(len(dataset["test"])) + len(dataset["train"]) + + len(dataset["validation"]), + } + + if self.limit >= 0: + self.raw_dataset = self.raw_dataset.select(range(self.limit)) + self._save_raw_data() + + def _get_trips(self) -> Iterator[TripletLike]: + return chain.from_iterable( + (iter(ds["graph"]) for ds in self.raw_dataset)) + + def _build_graph(self) -> None: + trips = self._get_trips() + self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( + trips, pre_transform=preprocess_triplet) + + # Nodes: + nodes = self.indexer.get_unique_node_features() + x = self.model.encode(nodes, batch_size=256) + self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) + + # Edges: + edges = self.indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) + edge_attr = self.model.encode(edges, batch_size=256) + self.indexer.add_edge_feature( + new_feature_name="edge_attr", + new_feature_vals=edge_attr, + map_from_feature=EDGE_RELATION, + ) + + self.indexer.save(self.processed_paths[-1]) + + def _retrieve_subgraphs(self) -> None: + print("Encoding questions...") + self.questions = [ds["question"] for ds in self.raw_dataset] + q_embs = self.model.encode(self.questions, batch_size=256) + list_of_graphs = [] + self.raw_graphs = [] + print("Retrieving subgraphs...") + textual_nodes = self.textual_nodes + textual_edges = self.textual_edges + for index in tqdm(range(len(self.raw_dataset))): + data_i = self.raw_dataset[index] + local_trips = data_i["graph"] + if self.whole_graph_retrieval: + graph = self.indexer.to_data(node_feature_name="x", + edge_feature_name="edge_attr") + else: + graph = get_features_for_triplets( + self.indexer, local_trips, + pre_transform=preprocess_triplet) + textual_nodes = self.textual_nodes.iloc[ + graph["node_idx"]].reset_index() + textual_edges = self.textual_edges.iloc[ + graph["edge_idx"]].reset_index() + self.raw_graphs.append(graph) + pcst_subgraph, desc = retrieval_via_pcst( + graph, + q_embs[index], + textual_nodes, + textual_edges, + topk=3, + topk_e=5, + cost_e=0.5, + ) + question = f"Question: {data_i['question']}\nAnswer: " + label = ("|").join(data_i["answer"]).lower() + + pcst_subgraph["question"] = question + pcst_subgraph["label"] = label + pcst_subgraph["desc"] = desc + list_of_graphs.append(pcst_subgraph.to("cpu")) + torch.save(self.raw_graphs, self.processed_paths[-2]) + self.save(list_of_graphs, self.processed_paths[0]) + + def process(self) -> None: + self._load_raw_data() + self.model = SentenceTransformer().to(self.device) + self.model.eval() + if not os.path.exists(self.processed_dir[-1]): + print("Encoding graph...") + self._build_graph() + else: + print("Loading graph...") + self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) + self.textual_nodes = DataFrame.from_dict( + {"node_attr": self.indexer.get_node_features()}) + self.textual_nodes["node_id"] = self.textual_nodes.index + self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] + self.textual_edges = DataFrame(self.indexer.get_edge_features(), + columns=["src", "edge_attr", "dst"]) + self.textual_edges["src"] = [ + self.indexer._nodes[h] for h in self.textual_edges["src"] + ] + self.textual_edges["dst"] = [ + self.indexer._nodes[h] for h in self.textual_edges["dst"] + ] + self._retrieve_subgraphs() From 2117605d66d93d08d1c8d2b21ea99a4e6ce984e4 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:14:41 -0700 Subject: [PATCH 605/752] formatting --- torch_geometric/datasets/updated_web_qsp_dataset.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 1ec9c97b7347..bb4858ada87f 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -1,12 +1,18 @@ import os from itertools import chain -from typing import Iterator, List, no_type_check, Tuple +from typing import Iterator, List, Tuple, no_type_check import numpy as np import torch from tqdm import tqdm -from torch_geometric.data import InMemoryDataset, LargeGraphIndexer, TripletLike, get_features_for_triplets, Data +from torch_geometric.data import ( + Data, + InMemoryDataset, + LargeGraphIndexer, + TripletLike, + get_features_for_triplets, +) from torch_geometric.data.large_graph_indexer import EDGE_RELATION from torch_geometric.nn.nlp import SentenceTransformer @@ -31,6 +37,7 @@ except ImportError: WITH_DATASETS = False + @no_type_check def retrieval_via_pcst( graph: Data, From 866866cd4d2edf2c159e7f64dbeaa3e5894abe79 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:18:44 -0700 Subject: [PATCH 606/752] add dataset --- torch_geometric/datasets/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index cc8ef498debb..1c45d30abf21 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -77,6 +77,7 @@ from .myket import MyketDataset from .brca_tgca import BrcaTcga from .web_qsp_dataset import WebQSPDataset +from .updated_web_qsp_dataset import UpdatedWebQSPDataset from .dbp15k import DBP15K from .aminer import AMiner @@ -111,6 +112,7 @@ import torch_geometric.datasets.utils homo_datasets = [ + 'UpdatedWebQSPDataset', 'WebQSPDataset', 'KarateClub', 'TUDataset', From 70f0380c2e96ab02d727a49f9c520f4dcda41a5d Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:26:11 -0700 Subject: [PATCH 607/752] add option for new dataloader --- examples/llm_plus_gnn/g_retriever.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 7bc869c5e776..1a97f69dece6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -18,7 +18,7 @@ from tqdm import tqdm from torch_geometric import seed_everything -from torch_geometric.datasets import WebQSPDataset +from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM @@ -441,6 +441,10 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, parser.add_argument( "--skip_pretrained_llm_eval", action="store_true", help="This flag will skip the evaluation of the pretrained LLM.") + parser.add_argument( + "--updated_qsp", action="store_true", + help="This enables the updated and inproved WebQSP dataloader." + ) args = parser.parse_args() # check if saved model @@ -453,12 +457,13 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, else: retrain = True if retrain: + dataset = UpdatedWebQSPDataset() if args.updated_qsp else None since = time.time() prep_time, dataset, gnn_llm_eval_outs = train( since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, args.batch_size, args.eval_batch_size, args.lr, get_loss, inference_step, checkpointing=args.checkpointing, - tiny_llama=args.tiny_llama) + tiny_llama=args.tiny_llama, dataset=dataset) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -467,7 +472,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") else: gnn_llm_eval_outs = torch.load("gnn_llm_eval_outs.pt") - dataset = WebQSPDataset() + dataset = WebQSPDataset() if not args.updated_qsp else UpdatedWebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, From db94397367724836d536c1b9b23db28dc312547b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:29:25 -0700 Subject: [PATCH 608/752] fix formatting --- examples/llm_plus_gnn/g_retriever.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1a97f69dece6..b70171b7758d 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -18,7 +18,7 @@ from tqdm import tqdm from torch_geometric import seed_everything -from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset +from torch_geometric.datasets import UpdatedWebQSPDataset, WebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM @@ -443,8 +443,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, help="This flag will skip the evaluation of the pretrained LLM.") parser.add_argument( "--updated_qsp", action="store_true", - help="This enables the updated and inproved WebQSP dataloader." - ) + help="This enables the updated and inproved WebQSP dataloader.") args = parser.parse_args() # check if saved model @@ -472,7 +471,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") else: gnn_llm_eval_outs = torch.load("gnn_llm_eval_outs.pt") - dataset = WebQSPDataset() if not args.updated_qsp else UpdatedWebQSPDataset() + dataset = WebQSPDataset( + ) if not args.updated_qsp else UpdatedWebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, From a03028a3e19e2f32a1b230c039071475fd7a84fa Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 3 Jun 2024 19:41:06 -0700 Subject: [PATCH 609/752] instantiate ds in train func --- examples/llm_plus_gnn/g_retriever.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index b70171b7758d..9d649e4f4d8f 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -18,6 +18,7 @@ from tqdm import tqdm from torch_geometric import seed_everything +from torch_geometric.data import Dataset from torch_geometric.datasets import UpdatedWebQSPDataset, WebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever @@ -140,6 +141,9 @@ def adjust_learning_rate(param_group, LR, epoch): if dataset is None: dataset = WebQSPDataset() gc.collect() + elif not isinstance(dataset, Dataset) and callable(dataset): + dataset = dataset() + gc.collect() idx_split = dataset.split_idxs # Step 1: Build Node Classification Dataset @@ -456,7 +460,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, else: retrain = True if retrain: - dataset = UpdatedWebQSPDataset() if args.updated_qsp else None + dataset = UpdatedWebQSPDataset if args.updated_qsp else None since = time.time() prep_time, dataset, gnn_llm_eval_outs = train( since, args.epochs, args.gnn_hidden_channels, args.num_gnn_layers, From 0ec7aac52896b5a2477c58d19de5e18f13bd0eed Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 4 Jun 2024 14:04:40 -0700 Subject: [PATCH 610/752] Restore mapping attrs --- torch_geometric/data/large_graph_indexer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 1104ed9ac1c5..584163dd4e6a 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -473,5 +473,10 @@ def apply_transform(trips): edge_attr=edge_attr, num_nodes=len(node_feats), ) + # needed for mappings + data_obj[NODE_PID] = node_keys + data_obj[EDGE_PID] = edge_keys + data_obj["node_idx"] = [indexer._nodes[k] for k in node_keys] + data_obj["edge_idx"] = [indexer._edges[e] for e in edge_keys] return data_obj From c1e5cc80b6b0e10ecb1357cd7374dfda40d76432 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 4 Jun 2024 14:12:45 -0700 Subject: [PATCH 611/752] Test edited to cover restoration of data info --- test/data/test_large_graph_indexer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index f9a6eebf573c..bf3f51fad691 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -11,7 +11,7 @@ TripletLike, get_features_for_triplets, ) -from torch_geometric.data.large_graph_indexer import EDGE_RELATION, NODE_PID +from torch_geometric.data.large_graph_indexer import EDGE_RELATION, NODE_PID, EDGE_PID # create possible nodes and edges for graph strkeys = string.ascii_letters + string.digits @@ -117,6 +117,12 @@ def encode_graph_from_trips(triplets: List[TripletLike]) -> Data: for g in graphs ] + for ds in large_graph_ds: + assert NODE_PID in ds + assert EDGE_PID in ds + assert "node_idx" in ds + assert "edge_idx" in ds + def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.99): def _sorted_tensors_are_close(tensor1, tensor2): From 5a0988128be16e6af3e197dedaec12be66505233 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 4 Jun 2024 19:21:50 -0700 Subject: [PATCH 612/752] begin trying profiling --- examples/llm_plus_gnn/test_profiling.ipynb | 143 +++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 examples/llm_plus_gnn/test_profiling.ipynb diff --git a/examples/llm_plus_gnn/test_profiling.ipynb b/examples/llm_plus_gnn/test_profiling.ipynb new file mode 100644 index 000000000000..2ff5474078b6 --- /dev/null +++ b/examples/llm_plus_gnn/test_profiling.ipynb @@ -0,0 +1,143 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from torch_geometric.profile import profileit, timeit\n", + "from torch_geometric.profile.profile import GPUStats\n", + "from typing import Protocol, Type, List, Tuple, Any\n", + "from abc import abstractmethod\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "class Profilable(Protocol):\n", + " model: torch.nn.Module\n", + " device: torch.device\n", + "\n", + " @abstractmethod\n", + " def _build_graph(self) -> None:\n", + " pass\n", + "\n", + " @abstractmethod\n", + " def _retrieve_subgraphs(self) -> None:\n", + " pass\n", + "\n", + "def make_profilable(dataset_obj: Type[Profilable]) -> Type[Profilable]:\n", + " dec = profileit(\"cuda\")\n", + "\n", + " class ProfilableObject(dataset_obj):\n", + " def __init__(self, *args, **kwargs) -> None:\n", + " self.desc = dict()\n", + " self.parent_cls = super()\n", + " self.parent_cls.__init__(*args, **kwargs)\n", + "\n", + " def _build_graph(self) -> None:\n", + " device_tensor = torch.Tensor().to(self.device)\n", + " wrap = dec(lambda model, dev_tensor: self.parent_cls._build_graph())\n", + " ret, desc = wrap(self.model, device_tensor)\n", + " self.desc['_build_graph'] = desc\n", + " return ret\n", + " \n", + " def _retrieve_subgraphs(self) -> None:\n", + " device_tensor = torch.Tensor().to(self.device)\n", + " wrap = dec(lambda model, dev_tensor: self.parent_cls._retrieve_subgraphs())\n", + " ret, desc = wrap(self.model, device_tensor)\n", + " self.desc['_retrieve_subgraphs'] = desc\n", + " return ret\n", + " \n", + " return ProfilableObject" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "profilable_ds: Type[UpdatedWebQSPDataset] = make_profilable(UpdatedWebQSPDataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Encoding graph...\n" + ] + }, + { + "ename": "TypeError", + "evalue": "super(type, obj): obj must be an instance or subtype of type", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[21], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m dataset: UpdatedWebQSPDataset \u001b[38;5;241m=\u001b[39m \u001b[43mprofilable_ds\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprofiled_ds\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[18], line 20\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject.__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m()\n\u001b[1;32m 19\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[0;32m---> 20\u001b[0m \u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:173\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, whole_graph_retrieval, limit)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwhole_graph_retrieval \u001b[38;5;241m=\u001b[39m whole_graph_retrieval\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\n\u001b[1;32m 172\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 173\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 174\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:306\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]):\n\u001b[1;32m 305\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncoding graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 306\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 307\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "Cell \u001b[0;32mIn[18], line 26\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject._build_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 24\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[1;32m 25\u001b[0m wrap \u001b[38;5;241m=\u001b[39m dec(\u001b[38;5;28;01mlambda\u001b[39;00m model, dev_tensor: \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m_build_graph())\n\u001b[0;32m---> 26\u001b[0m ret, desc \u001b[38;5;241m=\u001b[39m \u001b[43mwrap\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_tensor\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_build_graph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m desc\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ret\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/profile/profile.py:115\u001b[0m, in \u001b[0;36mprofileit..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m end \u001b[38;5;241m=\u001b[39m torch_gpu\u001b[38;5;241m.\u001b[39mEvent(enable_timing\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 113\u001b[0m start\u001b[38;5;241m.\u001b[39mrecord()\n\u001b[0;32m--> 115\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 117\u001b[0m end\u001b[38;5;241m.\u001b[39mrecord()\n\u001b[1;32m 118\u001b[0m torch_gpu\u001b[38;5;241m.\u001b[39msynchronize()\n", + "Cell \u001b[0;32mIn[18], line 25\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject._build_graph..\u001b[0;34m(model, dev_tensor)\u001b[0m\n\u001b[1;32m 23\u001b[0m device_tensor \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mTensor()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m 24\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[0;32m---> 25\u001b[0m wrap \u001b[38;5;241m=\u001b[39m dec(\u001b[38;5;28;01mlambda\u001b[39;00m model, dev_tensor: \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39m_build_graph())\n\u001b[1;32m 26\u001b[0m ret, desc \u001b[38;5;241m=\u001b[39m wrap(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, device_tensor)\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_build_graph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m desc\n", + "\u001b[0;31mTypeError\u001b[0m: super(type, obj): obj must be an instance or subtype of type" + ] + } + ], + "source": [ + "dataset: UpdatedWebQSPDataset = profilable_ds(root=\"profiled_ds\", force_reload=True, limit=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 9d1093cce5f0db5e4d9411bae22529ecb8b451de Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 5 Jun 2024 19:59:45 -0700 Subject: [PATCH 613/752] speedup retrieval to avoid timeouts --- torch_geometric/data/__init__.py | 4 +- torch_geometric/data/large_graph_indexer.py | 49 +++++++++++++++++++ .../datasets/updated_web_qsp_dataset.py | 30 ++++++------ 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/torch_geometric/data/__init__.py b/torch_geometric/data/__init__.py index 0a61bea38315..4489d46b2693 100644 --- a/torch_geometric/data/__init__.py +++ b/torch_geometric/data/__init__.py @@ -16,7 +16,7 @@ from .makedirs import makedirs from .download import download_url, download_google_url from .extract import extract_tar, extract_zip, extract_bz2, extract_gz -from .large_graph_indexer import LargeGraphIndexer, TripletLike, get_features_for_triplets +from .large_graph_indexer import LargeGraphIndexer, TripletLike, get_features_for_triplets, get_features_for_triplets_groups from torch_geometric.lazy_loader import LazyLoader @@ -47,7 +47,7 @@ helper_functions = [ 'makedirs', 'download_url', 'download_google_url', 'extract_tar', - 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets' + 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets', "get_features_for_triplets_groups" ] __all__ = data_classes + remote_backend_classes + helper_functions diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 584163dd4e6a..ac5417beb8c9 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -16,6 +16,7 @@ Set, Tuple, ) +from tqdm import tqdm import torch @@ -434,6 +435,49 @@ def to_data(self, node_feature_name: str, edge_feature_name) return Data(x=x, edge_index=edge_index, edge_attr=edge_attr) +def get_features_for_triplets_groups(indexer: LargeGraphIndexer, triplet_groups: Iterable[Iterable[TripletLike]], node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None) -> Iterator[Data]: + if pre_transform is not None: + def apply_transform(trips): + for trip in trips: + yield pre_transform(tuple(trip)) + + # TODO: Make this safe for large amounts of triplets? + triplet_groups = (list(apply_transform(triplets)) for triplets in triplet_groups) + + node_keys = [] + edge_keys = [] + edge_index = [] + + for triplets in tqdm(triplet_groups): + small_graph_indexer = LargeGraphIndexer.from_triplets(triplets, pre_transform=pre_transform) + + node_keys.append(small_graph_indexer.get_node_features()) + edge_keys.append(small_graph_indexer.get_edge_features(pids=triplets)) + edge_index.append(small_graph_indexer.get_edge_features(EDGE_INDEX, triplets)) + + node_feats = indexer.get_node_features(feature_name=node_feature_name, pids=chain.from_iterable(node_keys)) + edge_feats = indexer.get_edge_features(feature_name=edge_feature_name, pids=chain.from_iterable(edge_keys)) + + + last_node_idx, last_edge_idx = 0, 0 + for (nkeys, ekeys, eidx) in zip(node_keys, edge_keys, edge_index): + nlen, elen = len(nkeys), len(ekeys) + x = torch.Tensor(node_feats[last_node_idx:last_node_idx+nlen]) + last_node_idx += len(nkeys) + + edge_attr = torch.Tensor(edge_feats[last_edge_idx:last_edge_idx+elen]) + last_edge_idx += len(ekeys) + + edge_idx = torch.LongTensor(eidx).T + + data_obj = Data(x=x, edge_attr=edge_attr, edge_index=edge_idx) + data_obj[NODE_PID] = node_keys + data_obj[EDGE_PID] = edge_keys + data_obj["node_idx"] = [indexer._nodes[k] for k in nkeys] + data_obj["edge_idx"] = [indexer._edges[e] for e in ekeys] + + yield data_obj + def get_features_for_triplets( indexer: LargeGraphIndexer, @@ -443,6 +487,10 @@ def get_features_for_triplets( pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> Data: + gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, edge_feature_name, pre_transform) + return next(gen) + + ''' if pre_transform is not None: def apply_transform(trips): @@ -480,3 +528,4 @@ def apply_transform(trips): data_obj["edge_idx"] = [indexer._edges[e] for e in edge_keys] return data_obj + ''' diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index bb4858ada87f..e38301086720 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -11,7 +11,7 @@ InMemoryDataset, LargeGraphIndexer, TripletLike, - get_features_for_triplets, + get_features_for_triplets_groups ) from torch_geometric.data.large_graph_indexer import EDGE_RELATION from torch_geometric.nn.nlp import SentenceTransformer @@ -196,7 +196,6 @@ def processed_file_names(self) -> List[str]: "list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt", - "raw_graphs.pt", "large_graph_indexer", ] @@ -252,6 +251,7 @@ def _build_graph(self) -> None: map_from_feature=EDGE_RELATION, ) + print("Saving graph...") self.indexer.save(self.processed_paths[-1]) def _retrieve_subgraphs(self) -> None: @@ -259,25 +259,25 @@ def _retrieve_subgraphs(self) -> None: self.questions = [ds["question"] for ds in self.raw_dataset] q_embs = self.model.encode(self.questions, batch_size=256) list_of_graphs = [] - self.raw_graphs = [] print("Retrieving subgraphs...") textual_nodes = self.textual_nodes textual_edges = self.textual_edges + graph = None + graph_gen = None + if self.whole_graph_retrieval: + graph = self.indexer.to_data(node_feature_name="x", + edge_feature_name="edge_attr") + else: + graph_gen = get_features_for_triplets_groups(self.indexer, (ds['graph'] for ds in self.raw_dataset), pre_transform=preprocess_triplet) + for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] - local_trips = data_i["graph"] - if self.whole_graph_retrieval: - graph = self.indexer.to_data(node_feature_name="x", - edge_feature_name="edge_attr") - else: - graph = get_features_for_triplets( - self.indexer, local_trips, - pre_transform=preprocess_triplet) + if not self.whole_graph_retrieval: + graph = next(graph_gen) textual_nodes = self.textual_nodes.iloc[ graph["node_idx"]].reset_index() textual_edges = self.textual_edges.iloc[ graph["edge_idx"]].reset_index() - self.raw_graphs.append(graph) pcst_subgraph, desc = retrieval_via_pcst( graph, q_embs[index], @@ -294,19 +294,19 @@ def _retrieve_subgraphs(self) -> None: pcst_subgraph["label"] = label pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) - torch.save(self.raw_graphs, self.processed_paths[-2]) + print("Saving subgraphs...") self.save(list_of_graphs, self.processed_paths[0]) def process(self) -> None: self._load_raw_data() self.model = SentenceTransformer().to(self.device) self.model.eval() - if not os.path.exists(self.processed_dir[-1]): + if not os.path.exists(self.processed_paths[-1]): print("Encoding graph...") self._build_graph() else: print("Loading graph...") - self.indexer = LargeGraphIndexer.from_disk(self.processed_dir[-1]) + self.indexer = LargeGraphIndexer.from_disk(self.processed_paths[-1]) self.textual_nodes = DataFrame.from_dict( {"node_attr": self.indexer.get_node_features()}) self.textual_nodes["node_id"] = self.textual_nodes.index From 27acc5b8f9f3905c7d2fdf5de32cc7c22a17470a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 11 Jun 2024 14:48:17 -0700 Subject: [PATCH 614/752] init commit for retrieval --- examples/llm_plus_gnn/profiling_utils.py | 135 ++++++++++++++++++++ examples/llm_plus_gnn/rag_feature_store.py | 55 ++++++++ examples/llm_plus_gnn/rag_graph_store.py | 61 +++++++++ examples/llm_plus_gnn/rag_loader.py | 55 ++++++++ torch_geometric/data/large_graph_indexer.py | 61 ++------- 5 files changed, 318 insertions(+), 49 deletions(-) create mode 100644 examples/llm_plus_gnn/profiling_utils.py create mode 100644 examples/llm_plus_gnn/rag_feature_store.py create mode 100644 examples/llm_plus_gnn/rag_graph_store.py create mode 100644 examples/llm_plus_gnn/rag_loader.py diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py new file mode 100644 index 000000000000..144ff29d4835 --- /dev/null +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -0,0 +1,135 @@ +from torch_geometric.data import LargeGraphIndexer, FeatureStore, GraphStore, TripletLike +from torch_geometric.data.large_graph_indexer import EDGE_RELATION +from torch_geometric.distributed import LocalFeatureStore, LocalGraphStore, Partitioner +from typing import Protocol, Iterable, Optional, Tuple, Type, Callable, runtime_checkable, Dict +from torch_geometric.typing import EdgeType, NodeType +from torch import Tensor, Module +from dataclasses import dataclass +from enum import Enum, auto +import torch + +RemoteGraphBackend = Tuple[GraphStore, FeatureStore] + +# TODO: Make everything compatible with Hetero graphs aswell + + + +# Adapted from LocalGraphStore +@runtime_checkable +class ConvertableGraphStore(Protocol, GraphStore): + @classmethod + def from_data( + cls, + edge_id: Tensor, + edge_index: Tensor, + num_nodes: int, + is_sorted: bool = False, + ) -> GraphStore: + ... + + @classmethod + def from_hetero_data( + cls, + edge_id_dict: Dict[EdgeType, Tensor], + edge_index_dict: Dict[EdgeType, Tensor], + num_nodes_dict: Dict[NodeType, int], + is_sorted: bool = False, + ) -> GraphStore: + ... + + @classmethod + def from_partition(cls, root: str, pid: int) -> GraphStore: + ... + +# Adapted from LocalFeatureStore +@runtime_checkable +class ConvertableFeatureStore(Protocol, FeatureStore): + @classmethod + def from_data( + cls, + node_id: Tensor, + x: Optional[Tensor] = None, + y: Optional[Tensor] = None, + edge_id: Optional[Tensor] = None, + edge_attr: Optional[Tensor] = None, + ) -> FeatureStore: + ... + + @classmethod + def from_hetero_data( + cls, + node_id_dict: Dict[NodeType, Tensor], + x_dict: Optional[Dict[NodeType, Tensor]] = None, + y_dict: Optional[Dict[NodeType, Tensor]] = None, + edge_id_dict: Optional[Dict[EdgeType, Tensor]] = None, + edge_attr_dict: Optional[Dict[EdgeType, Tensor]] = None, + ) -> FeatureStore: + ... + + @classmethod + def from_partition(cls, root: str, pid: int) -> FeatureStore: + ... + +class RemoteDataType(Enum): + DATA = auto() + PARTITION = auto() + +@dataclass +class RemoteGraphBackendLoader: + path: str + datatype: RemoteDataType + graph_store_type: Type[ConvertableGraphStore] + feature_store_type: Type[ConvertableFeatureStore] + + def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: + if self.datatype == RemoteDataType.DATA: + data_obj = torch.load(self.path) + graph_store = self.graph_store_type.from_data(edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, num_nodes=data_obj.num_nodes, is_sorted=True) + feature_store = self.feature_store_type.from_data(node_id=data_obj['node_id'], x=data_obj.x, edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) + elif self.datatype == RemoteDataType.PARTITION: + if pid is None: + assert pid is not None, "Partition ID must be defined for loading from a partitioned store." + graph_store = self.graph_store_type.from_partition(self.path, pid) + feature_store = self.feature_store_type.from_partition(self.path, pid) + else: + raise NotImplementedError + return (graph_store, feature_store) + +# TODO: make profilable +def create_remote_backend_from_triplets(triplets: Iterable[TripletLike], node_embedding_model: Module, edge_embedding_model: Module | None = None, graph_db: Type[ConvertableGraphStore] = LocalGraphStore, feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, node_method_to_call: str = "forward", edge_method_to_call: str | None = None, pre_transform: Callable[[TripletLike], TripletLike] | None = None, path: str = '', n_parts: int = 1) -> RemoteGraphBackendLoader: + + # Will return attribute errors for missing attributes + if not issubclass(graph_db, ConvertableGraphStore): + getattr(graph_db, "from_data") + getattr(graph_db, "from_hetero_data") + getattr(graph_db, "from_partition") + elif not issubclass(feature_db, ConvertableFeatureStore): + getattr(feature_db, "from_data") + getattr(feature_db, "from_hetero_data") + getattr(feature_db, "from_partition") + + # Resolve callable methods + edge_embedding_model = edge_embedding_model if edge_embedding_model is not None else node_embedding_model + edge_method_to_call = edge_method_to_call if edge_method_to_call is not None else node_method_to_call + + # These will return AttributeErrors if they don't exist + node_model = getattr(node_embedding_model, node_method_to_call) + edge_model = getattr(edge_embedding_model, edge_method_to_call) + + indexer = LargeGraphIndexer.from_triplets(triplets) + + node_feats = node_model(indexer.get_node_features()) + indexer.add_node_feature('x', node_feats) + + edge_feats = edge_model(indexer.get_edge_features(feature_name=EDGE_RELATION)) + indexer.add_edge_feature('edge_attr', edge_feats, map_from_feature=EDGE_RELATION) + + data = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') + + if n_parts == 1: + torch.save(data, path) + return RemoteGraphBackendLoader(path, RemoteDataType.DATA, graph_db, feature_db) + else: + partitioner = Partitioner(data=data, num_parts=n_parts, root=path) + partitioner.generate_partition() + return RemoteGraphBackendLoader(path, RemoteDataType.PARTITION, graph_db, feature_db) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py new file mode 100644 index 000000000000..928163f86b56 --- /dev/null +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -0,0 +1,55 @@ +from torch_geometric.distributed import LocalFeatureStore +from torch import Tensor +from torch.nn import Module +from torch_geometric.nn.nlp import SentenceTransformer +import torch +from typing import Iterable, Type +from torchmetrics.functional import pairwise_cosine_similarity + + +# NOTE: Only compatible with Homogeneous graphs for now +class KNNRAGFeatureStore(LocalFeatureStore): + + def __init__(self, enc_model: Type[Module], *args, **kwargs): + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.enc_model = enc_model(*args, **kwargs).to(self.device) + self.enc_model.eval() + super().__init__() + + @property + def x(self) -> Tensor: + return self.get_tensor(group_name=None, attr_name='x') + + @property + def edge_attr(self) -> Tensor: + return self.get_tensor(group_name=(None, None), attr_name='edge_attr') + + @property + def edge_id(self) -> Tensor: + return self.get_tensor(group_name=(None, None)) + + def retrieve_seed_nodes(self, query: Iterable[str], k_nodes: int = 5) -> Tensor: + query_enc = self.enc_model.encode(query) + prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) + topk = min(k_nodes, len(self.x)) + topk_n_indices = [] + for q in prizes: + _, indices = torch.topk(q, topk, largest=True) + topk_n_indices.append(indices) + + return torch.Tensor(topk_n_indices) + + def retrieve_seed_edges(self, query: Iterable[str], k_edges: int = 3) -> Tensor: + query_enc = self.enc_model.encode(query) + prizes = pairwise_cosine_similarity(query_enc, self.edge_attr.to(self.device)) + topk = min(k_edges, len(self.edge_attr)) + topk_n_indices = [] + for q in prizes: + _, indices = torch.topk(q, topk, largest=True) + topk_n_indices.append(indices) + return torch.Tensor(topk_n_indices) + + +class SentenceTransformerFeatureStore(KNNRAGFeatureStore): + def __init__(self, *args, **kwargs): + super().__init__(SentenceTransformer, *args, **kwargs) \ No newline at end of file diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py new file mode 100644 index 000000000000..cfc3ed9203ce --- /dev/null +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -0,0 +1,61 @@ +from torch_geometric.distributed import LocalGraphStore +from typing import Type, Protocol, Tuple, Union +from torch_geometric.sampler import BaseSampler, NeighborSampler +from torch_geometric.data import FeatureStore, GraphStore, Data, HeteroData +from torch_geometric.loader import NodeLoader +from abc import abstractmethod +from torch_geometric.typing import InputNodes, InputEdges +from torch import Tensor +from torch_geometric.sampler.neighbor_sampler import NumNeighborsType +import torch + +class DataSampler(Protocol, BaseSampler): + + @abstractmethod + def __init__(self, data: Union[Data, HeteroData, Tuple[FeatureStore, GraphStore]]) -> None: + ... + +class NeighborSamplingRAGGraphStore(LocalGraphStore): + def __init__(self, feature_store: FeatureStore, num_neighbors: NumNeighborsType = [10], **kwargs): + self.feature_store = feature_store + self.num_neighbors = num_neighbors + self.sample_kwargs = kwargs + self._sampler_is_initialized = False + super().__init__() + + def _init_sampler(self): + self.sampler = NeighborSampler(data=(self.feature_store, self), num_neighbors=self.num_neighbors, **self.sample_kwargs) + self._sampler_is_initialized = True + + def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: + ret = super().put_edge_id(edge_id, *args, **kwargs) + self._sampler_is_initialized = False + return ret + + def put_edge_index(self, edge_index: Tuple[Tensor], *args, **kwargs) -> bool: + ret = super().put_edge_index(edge_index, *args, **kwargs) + self._sampler_is_initialized = False + return ret + + + def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: + if not self._sampler_is_initialized: + self._init_sampler() + + # Right now, only input nodes as tensors will be supported + if not isinstance(seed_nodes, Tensor): + raise NotImplementedError + + if seed_edges is not None: + if isinstance(seed_edges, Tensor): + seed_edges = seed_edges.reshape((-1)) + seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) + else: + raise NotImplementedError + + seed_nodes = seed_nodes.unique() + num_nodes = len(seed_nodes) + loader = NodeLoader(data=(self.feature_store, self), node_sampler=self.sampler, input_nodes=seed_nodes, batch_size=num_nodes) + + return next(iter(loader)) + \ No newline at end of file diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py new file mode 100644 index 000000000000..bab810e9d1c4 --- /dev/null +++ b/examples/llm_plus_gnn/rag_loader.py @@ -0,0 +1,55 @@ +from torch_geometric.data import FeatureStore, GraphStore, Data, HeteroData +from typing import Protocol, Any, Tuple, Union, Optional, Callable +from torch_geometric.typing import EdgeTensorType, InputNodes, InputEdges +from abc import abstractmethod +from torch import Tensor +import torch + + +class RAGFeatureStore(Protocol, FeatureStore): + """Feature store for remote GNN RAG backend.""" + + @abstractmethod + def retrieve_seed_nodes(self, query: Any, **kwargs) -> Tensor: + """Makes a comparison between the query and all the nodes to get all the closest nodes. Return the indices of the nodes that are to be seeds for the RAG Sampler.""" + ... + + @abstractmethod + def retieve_seed_edges(self, query: Any, **kwargs) -> EdgeTensorType: + """Makes a comparison between the query and all the edges to get all the closest nodes. Returns the edge indices that are to be the seeds for the RAG Sampler.""" + ... + + +class RAGGraphStore(Protocol, GraphStore): + """Graph store for remote GNN RAG backend.""" + + @abstractmethod + def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: + """Sample a subgraph using the seeded nodes and edges.""" + ... + +# TODO: Make compatible with Heterographs + + +class RagQueryLoader: + def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], local_filter: Optional[Callable] = None, **kwargs): + fstore, gstore = data + self.feature_store: RAGFeatureStore = fstore + self.graph_store: RAGGraphStore = gstore + self.local_filter = local_filter + + def query(self, query: Any) -> Data: + """Retrieve a subgraph associated with the query with all its feature attributes.""" + seed_nodes = self.feature_store.retrieve_seed_nodes(query) + seed_edges = self.feature_store.retrieve_seed_edges(query) + + subgraph = self.graph_store.retrieve_subgraph(seed_nodes, seed_edges).edge_index + + nodes = torch.cat((subgraph[0], subgraph[1]), dim=0).unique() + x = self.feature_store.multi_get_tensor(nodes) + edge_attr = self.feature_store.multi_get_tensor(subgraph) + + data = Data(x=x, edge_attr=edge_attr, edge_index=subgraph) + if self.local_filter: + data = self.local_filter(data) + return data diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index ac5417beb8c9..df83b42af53b 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -212,6 +212,8 @@ def get_node_features( values = self.node_attr[feature_name].values else: values = self.node_attr[feature_name] + + # TODO: torch_geometric.utils.select if isinstance(values, torch.Tensor): idxs = list( self.get_node_features_iter(feature_name, pids, @@ -300,6 +302,8 @@ def get_edge_features( values = self.edge_attr[feature_name].values else: values = self.edge_attr[feature_name] + + # TODO: torch_geometric.utils.select if isinstance(values, torch.Tensor): idxs = list( self.get_edge_features_iter(feature_name, pids, @@ -419,21 +423,20 @@ def __eq__(self, value: "LargeGraphIndexer") -> bool: eq &= self.edge_attr[k] == value.edge_attr[k] return eq - def _to_data_attrs( - self, node_feature_name: str, edge_feature_name: Optional[str] = None - ) -> Tuple[torch.Tensor, torch.LongTensor, Optional[torch.Tensor]]: + def to_data(self, node_feature_name: str, + edge_feature_name: Optional[str] = None) -> Data: x = torch.Tensor(self.get_node_features(node_feature_name)) + node_id = torch.LongTensor(range(len(x))) + edge_index = torch.t( torch.LongTensor(self.get_edge_features(EDGE_INDEX))) + edge_attr = (self.get_edge_features(edge_feature_name) if edge_feature_name is not None else None) - return x, edge_index, edge_attr + edge_id = torch.LongTensor(range(len(edge_attr))) + + return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, edge_id=edge_id, node_id=node_id) - def to_data(self, node_feature_name: str, - edge_feature_name: Optional[str] = None) -> Data: - x, edge_index, edge_attr = self._to_data_attrs(node_feature_name, - edge_feature_name) - return Data(x=x, edge_index=edge_index, edge_attr=edge_attr) def get_features_for_triplets_groups(indexer: LargeGraphIndexer, triplet_groups: Iterable[Iterable[TripletLike]], node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None) -> Iterator[Data]: if pre_transform is not None: @@ -489,43 +492,3 @@ def get_features_for_triplets( gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, edge_feature_name, pre_transform) return next(gen) - - ''' - if pre_transform is not None: - - def apply_transform(trips): - for trip in trips: - yield pre_transform(tuple(trip)) - - # TODO: Make this safe for large amounts of triplets? - triplets = list(apply_transform(triplets)) - - small_graph_indexer = LargeGraphIndexer.from_triplets( - triplets, pre_transform=pre_transform) - node_keys = small_graph_indexer.get_node_features() - node_feats = indexer.get_node_features(feature_name=node_feature_name, - pids=node_keys) - - edge_keys = small_graph_indexer.get_edge_features(pids=triplets) - edge_feats = indexer.get_edge_features(feature_name=edge_feature_name, - pids=edge_keys) - - edge_index = small_graph_indexer.get_edge_features(EDGE_INDEX, triplets) - - x = torch.Tensor(node_feats) - edge_attr = torch.Tensor(edge_feats) - edge_index = torch.t(torch.LongTensor(edge_index)) - data_obj = Data( - x=x, - edge_index=edge_index, - edge_attr=edge_attr, - num_nodes=len(node_feats), - ) - # needed for mappings - data_obj[NODE_PID] = node_keys - data_obj[EDGE_PID] = edge_keys - data_obj["node_idx"] = [indexer._nodes[k] for k in node_keys] - data_obj["edge_idx"] = [indexer._edges[e] for e in edge_keys] - - return data_obj - ''' From ff24ce9d4805396868c43269b24eab29e82c9152 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 11 Jun 2024 14:49:13 -0700 Subject: [PATCH 615/752] profiling nb --- examples/llm_plus_gnn/test_profiling.ipynb | 376 +++++++++++++++++++-- 1 file changed, 353 insertions(+), 23 deletions(-) diff --git a/examples/llm_plus_gnn/test_profiling.ipynb b/examples/llm_plus_gnn/test_profiling.ipynb index 2ff5474078b6..153e628407e9 100644 --- a/examples/llm_plus_gnn/test_profiling.ipynb +++ b/examples/llm_plus_gnn/test_profiling.ipynb @@ -2,13 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from raw_qsp_dataset import RawWebQSPDataset\n", "from torch_geometric.profile import profileit, timeit\n", "from torch_geometric.profile.profile import GPUStats\n", + "from torch.profiler import profile\n", "from typing import Protocol, Type, List, Tuple, Any\n", "from abc import abstractmethod\n", "import torch" @@ -16,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -60,7 +62,25 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds = UpdatedWebQSPDataset(root=\"profiled_ds\", force_reload=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds = UpdatedWebQSPDataset(root=\"profiled_ds_wholegraph\", force_reload=True, whole_graph_retrieval=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +89,122 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset: UpdatedWebQSPDataset = profilable_ds(root=\"profiled_ds\", force_reload=True, limit=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset.desc['_retrieve_subgraphs']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with profile(profile_memory=True, with_stack=True, record_shapes=True) as prof:\n", + " ds = UpdatedWebQSPDataset(root=\"profiled_ds\", force_reload=True, limit=10)\n", + " del ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prof.export_chrome_trace('timeline.json')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prof.export_memory_timeline('timeline_mem.html')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "avgs = prof.key_averages()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(avgs.table(sort_by='cpu_time'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torchmetrics.functional import pairwise_cosine_similarity\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "i1 = torch.rand((4700,1024)).to(device)\n", + "i2 = torch.rand((1000000,1024)).to(device)\n", + "i1 = pairwise_cosine_similarity(i1, i2)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from torch_geometric.data import LargeGraphIndexer\n", + "from itertools import chain" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -83,32 +218,227 @@ "name": "stdout", "output_type": "stream", "text": [ - "Encoding graph...\n" + "Loading graph...\n", + "Encoding questions...\n", + "Retrieving subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 59.05it/s]00:00 1\u001b[0m dataset: UpdatedWebQSPDataset \u001b[38;5;241m=\u001b[39m \u001b[43mprofilable_ds\u001b[49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprofiled_ds\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[18], line 20\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject.__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m()\n\u001b[1;32m 19\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[0;32m---> 20\u001b[0m \u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:173\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, whole_graph_retrieval, limit)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwhole_graph_retrieval \u001b[38;5;241m=\u001b[39m whole_graph_retrieval\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\n\u001b[1;32m 172\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 173\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 174\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:306\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]):\n\u001b[1;32m 305\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncoding graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 306\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 307\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "Cell \u001b[0;32mIn[18], line 26\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject._build_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 24\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[1;32m 25\u001b[0m wrap \u001b[38;5;241m=\u001b[39m dec(\u001b[38;5;28;01mlambda\u001b[39;00m model, dev_tensor: \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m_build_graph())\n\u001b[0;32m---> 26\u001b[0m ret, desc \u001b[38;5;241m=\u001b[39m \u001b[43mwrap\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_tensor\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_build_graph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m desc\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ret\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/profile/profile.py:115\u001b[0m, in \u001b[0;36mprofileit..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m end \u001b[38;5;241m=\u001b[39m torch_gpu\u001b[38;5;241m.\u001b[39mEvent(enable_timing\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 113\u001b[0m start\u001b[38;5;241m.\u001b[39mrecord()\n\u001b[0;32m--> 115\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 117\u001b[0m end\u001b[38;5;241m.\u001b[39mrecord()\n\u001b[1;32m 118\u001b[0m torch_gpu\u001b[38;5;241m.\u001b[39msynchronize()\n", - "Cell \u001b[0;32mIn[18], line 25\u001b[0m, in \u001b[0;36mmake_profilable..ProfilableObject._build_graph..\u001b[0;34m(model, dev_tensor)\u001b[0m\n\u001b[1;32m 23\u001b[0m device_tensor \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mTensor()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m 24\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\n\u001b[0;32m---> 25\u001b[0m wrap \u001b[38;5;241m=\u001b[39m dec(\u001b[38;5;28;01mlambda\u001b[39;00m model, dev_tensor: \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39m_build_graph())\n\u001b[1;32m 26\u001b[0m ret, desc \u001b[38;5;241m=\u001b[39m wrap(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, device_tensor)\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdesc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_build_graph\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m desc\n", - "\u001b[0;31mTypeError\u001b[0m: super(type, obj): obj must be an instance or subtype of type" + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Done!\n" ] } ], "source": [ - "dataset: UpdatedWebQSPDataset = profilable_ds(root=\"profiled_ds\", force_reload=True, limit=2)" + "ds = UpdatedWebQSPDataset('small_ds', force_reload=True, limit=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "graph = ds.indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "graph.edge_index = graph.edge_index.contiguous()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.loader import LinkNeighborLoader, LinkLoader\n", + "from torch_geometric.sampler import NeighborSampler, BaseSampler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.sampler.base import EdgeSamplerInput, HeteroSamplerOutput, NegativeSampling, NodeSamplerInput, SamplerOutput\n", + "\n", + "\n", + "class IdentitySampler(BaseSampler):\n", + " def sample_from_nodes(self, index: NodeSamplerInput, **kwargs) -> HeteroSamplerOutput | SamplerOutput:\n", + " if index.input_type is not None: # Heterogeneous\n", + " out = HeteroSamplerOutput(node={index.input_type: index.node}, row=dict(), col=dict(), edge=dict())\n", + " else:\n", + " out = SamplerOutput(node=index.node, row=torch.Tensor(), col=torch.Tensor())\n", + " return out\n", + " \n", + " def sample_from_edges(self, index: EdgeSamplerInput, neg_sampling: NegativeSampling | None = None) -> HeteroSamplerOutput | SamplerOutput:\n", + " EdgeSamplerInput()\n", + " if index.input_type is not None: # Heterogeneous\n", + " out = HeteroSamplerOutput(node=index.)" + ] + }, + { + "cell_type": "code", + "execution_count": 141, + "metadata": {}, + "outputs": [], + "source": [ + "link_sampler = NeighborSampler(data=graph, num_neighbors=[1], replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 142, + "metadata": {}, + "outputs": [], + "source": [ + "load = LinkLoader(graph, link_sampler=link_sampler)" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "11466" + ] + }, + "execution_count": 159, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "link_sampler.num_nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "load2 = LinkNeighborLoader(graph, num_neighbors=[2])" + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "metadata": {}, + "outputs": [], + "source": [ + "result = load([1])" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([17505, 24971])" + ] + }, + "execution_count": 154, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.e_id" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[10097, 4044, 5397, ..., 7188, 8251, 3597],\n", + " [10097, 673, 5827, ..., 7594, 673, 10951]])" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "graph.edge_index" + ] + }, + { + "cell_type": "code", + "execution_count": 153, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'m.09g3c50'" + ] + }, + "execution_count": 153, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.indexer.node_attr['pid'][4759]" + ] + }, + { + "cell_type": "code", + "execution_count": 155, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('washington redskins at oakland raiders, 2009-12-13',\n", + " 'american_football.football_game.away_team',\n", + " 'washington redskins')" + ] + }, + "execution_count": 155, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.indexer.edge_attr[\"e_pid\"][1]" ] }, { From 8180fa6ff34055aac8bd9982c490b02bd478ff40 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 11 Jun 2024 14:57:12 -0700 Subject: [PATCH 616/752] pre commit --- examples/llm_plus_gnn/profiling_utils.py | 87 ++++++++++++++----- examples/llm_plus_gnn/rag_feature_store.py | 27 +++--- examples/llm_plus_gnn/rag_graph_store.py | 52 ++++++----- examples/llm_plus_gnn/rag_loader.py | 38 +++++--- test/data/test_large_graph_indexer.py | 6 +- torch_geometric/data/__init__.py | 3 +- torch_geometric/data/large_graph_indexer.py | 46 ++++++---- .../datasets/updated_web_qsp_dataset.py | 11 ++- 8 files changed, 182 insertions(+), 88 deletions(-) diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py index 144ff29d4835..376be2ee1409 100644 --- a/examples/llm_plus_gnn/profiling_utils.py +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -1,18 +1,37 @@ -from torch_geometric.data import LargeGraphIndexer, FeatureStore, GraphStore, TripletLike -from torch_geometric.data.large_graph_indexer import EDGE_RELATION -from torch_geometric.distributed import LocalFeatureStore, LocalGraphStore, Partitioner -from typing import Protocol, Iterable, Optional, Tuple, Type, Callable, runtime_checkable, Dict -from torch_geometric.typing import EdgeType, NodeType -from torch import Tensor, Module from dataclasses import dataclass from enum import Enum, auto +from typing import ( + Callable, + Dict, + Iterable, + Optional, + Protocol, + Tuple, + Type, + runtime_checkable, +) + import torch +from torch import Module, Tensor + +from torch_geometric.data import ( + FeatureStore, + GraphStore, + LargeGraphIndexer, + TripletLike, +) +from torch_geometric.data.large_graph_indexer import EDGE_RELATION +from torch_geometric.distributed import ( + LocalFeatureStore, + LocalGraphStore, + Partitioner, +) +from torch_geometric.typing import EdgeType, NodeType RemoteGraphBackend = Tuple[GraphStore, FeatureStore] # TODO: Make everything compatible with Hetero graphs aswell - - + # Adapted from LocalGraphStore @runtime_checkable @@ -41,6 +60,7 @@ def from_hetero_data( def from_partition(cls, root: str, pid: int) -> GraphStore: ... + # Adapted from LocalFeatureStore @runtime_checkable class ConvertableFeatureStore(Protocol, FeatureStore): @@ -70,10 +90,12 @@ def from_hetero_data( def from_partition(cls, root: str, pid: int) -> FeatureStore: ... + class RemoteDataType(Enum): DATA = auto() PARTITION = auto() + @dataclass class RemoteGraphBackendLoader: path: str @@ -84,19 +106,35 @@ class RemoteGraphBackendLoader: def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: if self.datatype == RemoteDataType.DATA: data_obj = torch.load(self.path) - graph_store = self.graph_store_type.from_data(edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, num_nodes=data_obj.num_nodes, is_sorted=True) - feature_store = self.feature_store_type.from_data(node_id=data_obj['node_id'], x=data_obj.x, edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) + graph_store = self.graph_store_type.from_data( + edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, + num_nodes=data_obj.num_nodes, is_sorted=True) + feature_store = self.feature_store_type.from_data( + node_id=data_obj['node_id'], x=data_obj.x, + edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) elif self.datatype == RemoteDataType.PARTITION: if pid is None: - assert pid is not None, "Partition ID must be defined for loading from a partitioned store." + assert pid is not None, \ + "Partition ID must be defined for loading from a " \ + + "partitioned store." graph_store = self.graph_store_type.from_partition(self.path, pid) - feature_store = self.feature_store_type.from_partition(self.path, pid) + feature_store = self.feature_store_type.from_partition( + self.path, pid) else: raise NotImplementedError return (graph_store, feature_store) + # TODO: make profilable -def create_remote_backend_from_triplets(triplets: Iterable[TripletLike], node_embedding_model: Module, edge_embedding_model: Module | None = None, graph_db: Type[ConvertableGraphStore] = LocalGraphStore, feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, node_method_to_call: str = "forward", edge_method_to_call: str | None = None, pre_transform: Callable[[TripletLike], TripletLike] | None = None, path: str = '', n_parts: int = 1) -> RemoteGraphBackendLoader: +def create_remote_backend_from_triplets( + triplets: Iterable[TripletLike], node_embedding_model: Module, + edge_embedding_model: Module | None = None, + graph_db: Type[ConvertableGraphStore] = LocalGraphStore, + feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, + node_method_to_call: str = "forward", + edge_method_to_call: str | None = None, + pre_transform: Callable[[TripletLike], TripletLike] | None = None, + path: str = '', n_parts: int = 1) -> RemoteGraphBackendLoader: # Will return attribute errors for missing attributes if not issubclass(graph_db, ConvertableGraphStore): @@ -109,27 +147,34 @@ def create_remote_backend_from_triplets(triplets: Iterable[TripletLike], node_em getattr(feature_db, "from_partition") # Resolve callable methods - edge_embedding_model = edge_embedding_model if edge_embedding_model is not None else node_embedding_model - edge_method_to_call = edge_method_to_call if edge_method_to_call is not None else node_method_to_call + edge_embedding_model = edge_embedding_model \ + if edge_embedding_model is not None else node_embedding_model + edge_method_to_call = edge_method_to_call \ + if edge_method_to_call is not None else node_method_to_call # These will return AttributeErrors if they don't exist node_model = getattr(node_embedding_model, node_method_to_call) - edge_model = getattr(edge_embedding_model, edge_method_to_call) + edge_model = getattr(edge_embedding_model, edge_method_to_call) indexer = LargeGraphIndexer.from_triplets(triplets) node_feats = node_model(indexer.get_node_features()) indexer.add_node_feature('x', node_feats) - edge_feats = edge_model(indexer.get_edge_features(feature_name=EDGE_RELATION)) - indexer.add_edge_feature('edge_attr', edge_feats, map_from_feature=EDGE_RELATION) + edge_feats = edge_model( + indexer.get_edge_features(feature_name=EDGE_RELATION)) + indexer.add_edge_feature('edge_attr', edge_feats, + map_from_feature=EDGE_RELATION) - data = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') + data = indexer.to_data(node_feature_name='x', + edge_feature_name='edge_attr') if n_parts == 1: torch.save(data, path) - return RemoteGraphBackendLoader(path, RemoteDataType.DATA, graph_db, feature_db) + return RemoteGraphBackendLoader(path, RemoteDataType.DATA, graph_db, + feature_db) else: partitioner = Partitioner(data=data, num_parts=n_parts, root=path) partitioner.generate_partition() - return RemoteGraphBackendLoader(path, RemoteDataType.PARTITION, graph_db, feature_db) + return RemoteGraphBackendLoader(path, RemoteDataType.PARTITION, + graph_db, feature_db) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index 928163f86b56..f1fa23236381 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -1,21 +1,23 @@ -from torch_geometric.distributed import LocalFeatureStore +from typing import Iterable, Type + +import torch from torch import Tensor from torch.nn import Module -from torch_geometric.nn.nlp import SentenceTransformer -import torch -from typing import Iterable, Type from torchmetrics.functional import pairwise_cosine_similarity +from torch_geometric.distributed import LocalFeatureStore +from torch_geometric.nn.nlp import SentenceTransformer + # NOTE: Only compatible with Homogeneous graphs for now class KNNRAGFeatureStore(LocalFeatureStore): - def __init__(self, enc_model: Type[Module], *args, **kwargs): - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") self.enc_model = enc_model(*args, **kwargs).to(self.device) self.enc_model.eval() super().__init__() - + @property def x(self) -> Tensor: return self.get_tensor(group_name=None, attr_name='x') @@ -28,7 +30,8 @@ def edge_attr(self) -> Tensor: def edge_id(self) -> Tensor: return self.get_tensor(group_name=(None, None)) - def retrieve_seed_nodes(self, query: Iterable[str], k_nodes: int = 5) -> Tensor: + def retrieve_seed_nodes(self, query: Iterable[str], + k_nodes: int = 5) -> Tensor: query_enc = self.enc_model.encode(query) prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) topk = min(k_nodes, len(self.x)) @@ -39,9 +42,11 @@ def retrieve_seed_nodes(self, query: Iterable[str], k_nodes: int = 5) -> Tensor: return torch.Tensor(topk_n_indices) - def retrieve_seed_edges(self, query: Iterable[str], k_edges: int = 3) -> Tensor: + def retrieve_seed_edges(self, query: Iterable[str], + k_edges: int = 3) -> Tensor: query_enc = self.enc_model.encode(query) - prizes = pairwise_cosine_similarity(query_enc, self.edge_attr.to(self.device)) + prizes = pairwise_cosine_similarity(query_enc, + self.edge_attr.to(self.device)) topk = min(k_edges, len(self.edge_attr)) topk_n_indices = [] for q in prizes: @@ -52,4 +57,4 @@ def retrieve_seed_edges(self, query: Iterable[str], k_edges: int = 3) -> Tensor: class SentenceTransformerFeatureStore(KNNRAGFeatureStore): def __init__(self, *args, **kwargs): - super().__init__(SentenceTransformer, *args, **kwargs) \ No newline at end of file + super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index cfc3ed9203ce..3c6aeb5f5105 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -1,47 +1,56 @@ -from torch_geometric.distributed import LocalGraphStore -from typing import Type, Protocol, Tuple, Union -from torch_geometric.sampler import BaseSampler, NeighborSampler -from torch_geometric.data import FeatureStore, GraphStore, Data, HeteroData -from torch_geometric.loader import NodeLoader from abc import abstractmethod -from torch_geometric.typing import InputNodes, InputEdges +from typing import Protocol, Tuple, Union + +import torch from torch import Tensor + +from torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData +from torch_geometric.distributed import LocalGraphStore +from torch_geometric.loader import NodeLoader +from torch_geometric.sampler import BaseSampler, NeighborSampler from torch_geometric.sampler.neighbor_sampler import NumNeighborsType -import torch +from torch_geometric.typing import InputEdges, InputNodes -class DataSampler(Protocol, BaseSampler): +class DataSampler(Protocol, BaseSampler): @abstractmethod - def __init__(self, data: Union[Data, HeteroData, Tuple[FeatureStore, GraphStore]]) -> None: + def __init__( + self, data: Union[Data, HeteroData, Tuple[FeatureStore, + GraphStore]]) -> None: ... + class NeighborSamplingRAGGraphStore(LocalGraphStore): - def __init__(self, feature_store: FeatureStore, num_neighbors: NumNeighborsType = [10], **kwargs): + def __init__(self, feature_store: FeatureStore, + num_neighbors: NumNeighborsType = [10], **kwargs): self.feature_store = feature_store self.num_neighbors = num_neighbors self.sample_kwargs = kwargs self._sampler_is_initialized = False - super().__init__() - + super().__init__() + def _init_sampler(self): - self.sampler = NeighborSampler(data=(self.feature_store, self), num_neighbors=self.num_neighbors, **self.sample_kwargs) + self.sampler = NeighborSampler(data=(self.feature_store, self), + num_neighbors=self.num_neighbors, + **self.sample_kwargs) self._sampler_is_initialized = True - + def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: ret = super().put_edge_id(edge_id, *args, **kwargs) self._sampler_is_initialized = False return ret - - def put_edge_index(self, edge_index: Tuple[Tensor], *args, **kwargs) -> bool: + + def put_edge_index(self, edge_index: Tuple[Tensor], *args, + **kwargs) -> bool: ret = super().put_edge_index(edge_index, *args, **kwargs) self._sampler_is_initialized = False return ret - - def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: + def retrieve_subgraph(self, seed_nodes: InputNodes, + seed_edges: InputEdges) -> Union[Data, HeteroData]: if not self._sampler_is_initialized: self._init_sampler() - + # Right now, only input nodes as tensors will be supported if not isinstance(seed_nodes, Tensor): raise NotImplementedError @@ -55,7 +64,8 @@ def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> U seed_nodes = seed_nodes.unique() num_nodes = len(seed_nodes) - loader = NodeLoader(data=(self.feature_store, self), node_sampler=self.sampler, input_nodes=seed_nodes, batch_size=num_nodes) + loader = NodeLoader(data=(self.feature_store, self), + node_sampler=self.sampler, input_nodes=seed_nodes, + batch_size=num_nodes) return next(iter(loader)) - \ No newline at end of file diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py index bab810e9d1c4..dcc5b2bdf68c 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/examples/llm_plus_gnn/rag_loader.py @@ -1,49 +1,61 @@ -from torch_geometric.data import FeatureStore, GraphStore, Data, HeteroData -from typing import Protocol, Any, Tuple, Union, Optional, Callable -from torch_geometric.typing import EdgeTensorType, InputNodes, InputEdges from abc import abstractmethod -from torch import Tensor +from typing import Any, Callable, Optional, Protocol, Tuple, Union + import torch +from torch import Tensor + +from torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData +from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes class RAGFeatureStore(Protocol, FeatureStore): """Feature store for remote GNN RAG backend.""" - @abstractmethod def retrieve_seed_nodes(self, query: Any, **kwargs) -> Tensor: - """Makes a comparison between the query and all the nodes to get all the closest nodes. Return the indices of the nodes that are to be seeds for the RAG Sampler.""" + """Makes a comparison between the query and all the nodes to get all + the closest nodes. Return the indices of the nodes that are to be seeds + for the RAG Sampler. + """ ... @abstractmethod def retieve_seed_edges(self, query: Any, **kwargs) -> EdgeTensorType: - """Makes a comparison between the query and all the edges to get all the closest nodes. Returns the edge indices that are to be the seeds for the RAG Sampler.""" + """Makes a comparison between the query and all the edges to get all + the closest nodes. Returns the edge indices that are to be the seeds + for the RAG Sampler. + """ ... class RAGGraphStore(Protocol, GraphStore): """Graph store for remote GNN RAG backend.""" - @abstractmethod - def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: + def retrieve_subgraph(self, seed_nodes: InputNodes, + seed_edges: InputEdges) -> Union[Data, HeteroData]: """Sample a subgraph using the seeded nodes and edges.""" ... + # TODO: Make compatible with Heterographs class RagQueryLoader: - def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], local_filter: Optional[Callable] = None, **kwargs): + def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], + local_filter: Optional[Callable] = None, **kwargs): fstore, gstore = data self.feature_store: RAGFeatureStore = fstore self.graph_store: RAGGraphStore = gstore self.local_filter = local_filter - + def query(self, query: Any) -> Data: - """Retrieve a subgraph associated with the query with all its feature attributes.""" + """Retrieve a subgraph associated with the query with all its feature + attributes. + """ seed_nodes = self.feature_store.retrieve_seed_nodes(query) seed_edges = self.feature_store.retrieve_seed_edges(query) - subgraph = self.graph_store.retrieve_subgraph(seed_nodes, seed_edges).edge_index + subgraph = self.graph_store.retrieve_subgraph(seed_nodes, + seed_edges).edge_index nodes = torch.cat((subgraph[0], subgraph[1]), dim=0).unique() x = self.feature_store.multi_get_tensor(nodes) diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index bf3f51fad691..4a5ba9ec56ee 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -11,7 +11,11 @@ TripletLike, get_features_for_triplets, ) -from torch_geometric.data.large_graph_indexer import EDGE_RELATION, NODE_PID, EDGE_PID +from torch_geometric.data.large_graph_indexer import ( + EDGE_PID, + EDGE_RELATION, + NODE_PID, +) # create possible nodes and edges for graph strkeys = string.ascii_letters + string.digits diff --git a/torch_geometric/data/__init__.py b/torch_geometric/data/__init__.py index 4489d46b2693..5acf82c22467 100644 --- a/torch_geometric/data/__init__.py +++ b/torch_geometric/data/__init__.py @@ -47,7 +47,8 @@ helper_functions = [ 'makedirs', 'download_url', 'download_google_url', 'extract_tar', - 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets', "get_features_for_triplets_groups" + 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets', + "get_features_for_triplets_groups" ] __all__ = data_classes + remote_backend_classes + helper_functions diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index df83b42af53b..cf1d777a57c6 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -16,9 +16,9 @@ Set, Tuple, ) -from tqdm import tqdm import torch +from tqdm import tqdm from torch_geometric.data import Data @@ -212,7 +212,7 @@ def get_node_features( values = self.node_attr[feature_name].values else: values = self.node_attr[feature_name] - + # TODO: torch_geometric.utils.select if isinstance(values, torch.Tensor): idxs = list( @@ -302,7 +302,7 @@ def get_edge_features( values = self.edge_attr[feature_name].values else: values = self.edge_attr[feature_name] - + # TODO: torch_geometric.utils.select if isinstance(values, torch.Tensor): idxs = list( @@ -435,40 +435,52 @@ def to_data(self, node_feature_name: str, if edge_feature_name is not None else None) edge_id = torch.LongTensor(range(len(edge_attr))) - return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, edge_id=edge_id, node_id=node_id) + return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + edge_id=edge_id, node_id=node_id) -def get_features_for_triplets_groups(indexer: LargeGraphIndexer, triplet_groups: Iterable[Iterable[TripletLike]], node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None) -> Iterator[Data]: +def get_features_for_triplets_groups( + indexer: LargeGraphIndexer, + triplet_groups: Iterable[Iterable[TripletLike]], + node_feature_name: str = "x", edge_feature_name: str = "edge_attr", + pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None +) -> Iterator[Data]: if pre_transform is not None: + def apply_transform(trips): for trip in trips: yield pre_transform(tuple(trip)) # TODO: Make this safe for large amounts of triplets? - triplet_groups = (list(apply_transform(triplets)) for triplets in triplet_groups) + triplet_groups = (list(apply_transform(triplets)) + for triplets in triplet_groups) node_keys = [] edge_keys = [] edge_index = [] - + for triplets in tqdm(triplet_groups): - small_graph_indexer = LargeGraphIndexer.from_triplets(triplets, pre_transform=pre_transform) + small_graph_indexer = LargeGraphIndexer.from_triplets( + triplets, pre_transform=pre_transform) node_keys.append(small_graph_indexer.get_node_features()) edge_keys.append(small_graph_indexer.get_edge_features(pids=triplets)) - edge_index.append(small_graph_indexer.get_edge_features(EDGE_INDEX, triplets)) - - node_feats = indexer.get_node_features(feature_name=node_feature_name, pids=chain.from_iterable(node_keys)) - edge_feats = indexer.get_edge_features(feature_name=edge_feature_name, pids=chain.from_iterable(edge_keys)) + edge_index.append( + small_graph_indexer.get_edge_features(EDGE_INDEX, triplets)) + node_feats = indexer.get_node_features(feature_name=node_feature_name, + pids=chain.from_iterable(node_keys)) + edge_feats = indexer.get_edge_features(feature_name=edge_feature_name, + pids=chain.from_iterable(edge_keys)) last_node_idx, last_edge_idx = 0, 0 for (nkeys, ekeys, eidx) in zip(node_keys, edge_keys, edge_index): nlen, elen = len(nkeys), len(ekeys) - x = torch.Tensor(node_feats[last_node_idx:last_node_idx+nlen]) + x = torch.Tensor(node_feats[last_node_idx:last_node_idx + nlen]) last_node_idx += len(nkeys) - - edge_attr = torch.Tensor(edge_feats[last_edge_idx:last_edge_idx+elen]) + + edge_attr = torch.Tensor(edge_feats[last_edge_idx:last_edge_idx + + elen]) last_edge_idx += len(ekeys) edge_idx = torch.LongTensor(eidx).T @@ -490,5 +502,7 @@ def get_features_for_triplets( pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> Data: - gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, edge_feature_name, pre_transform) + gen = get_features_for_triplets_groups(indexer, [triplets], + node_feature_name, + edge_feature_name, pre_transform) return next(gen) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index e38301086720..3946e0d2c3d5 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -11,7 +11,7 @@ InMemoryDataset, LargeGraphIndexer, TripletLike, - get_features_for_triplets_groups + get_features_for_triplets_groups, ) from torch_geometric.data.large_graph_indexer import EDGE_RELATION from torch_geometric.nn.nlp import SentenceTransformer @@ -266,9 +266,11 @@ def _retrieve_subgraphs(self) -> None: graph_gen = None if self.whole_graph_retrieval: graph = self.indexer.to_data(node_feature_name="x", - edge_feature_name="edge_attr") + edge_feature_name="edge_attr") else: - graph_gen = get_features_for_triplets_groups(self.indexer, (ds['graph'] for ds in self.raw_dataset), pre_transform=preprocess_triplet) + graph_gen = get_features_for_triplets_groups( + self.indexer, (ds['graph'] for ds in self.raw_dataset), + pre_transform=preprocess_triplet) for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] @@ -306,7 +308,8 @@ def process(self) -> None: self._build_graph() else: print("Loading graph...") - self.indexer = LargeGraphIndexer.from_disk(self.processed_paths[-1]) + self.indexer = LargeGraphIndexer.from_disk( + self.processed_paths[-1]) self.textual_nodes = DataFrame.from_dict( {"node_attr": self.indexer.get_node_features()}) self.textual_nodes["node_id"] = self.textual_nodes.index From 2e517038396b998c99c1372325679f3b435607c7 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 11 Jun 2024 19:21:21 -0700 Subject: [PATCH 617/752] debug part 1 --- examples/llm_plus_gnn/profiling_utils.py | 28 +-- examples/llm_plus_gnn/rag_feature_store.py | 26 ++- examples/llm_plus_gnn/rag_graph_store.py | 48 +++--- examples/llm_plus_gnn/rag_loader.py | 31 ++-- examples/llm_plus_gnn/test_rag.ipynb | 179 ++++++++++++++++++++ torch_geometric/data/large_graph_indexer.py | 2 +- 6 files changed, 258 insertions(+), 56 deletions(-) create mode 100644 examples/llm_plus_gnn/test_rag.ipynb diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py index 376be2ee1409..9b6b54b0055e 100644 --- a/examples/llm_plus_gnn/profiling_utils.py +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -9,10 +9,12 @@ Tuple, Type, runtime_checkable, + Any, ) import torch -from torch import Module, Tensor +from torch import Tensor +from torch.nn import Module from torch_geometric.data import ( FeatureStore, @@ -28,14 +30,14 @@ ) from torch_geometric.typing import EdgeType, NodeType -RemoteGraphBackend = Tuple[GraphStore, FeatureStore] +RemoteGraphBackend = Tuple[FeatureStore, GraphStore] # TODO: Make everything compatible with Hetero graphs aswell # Adapted from LocalGraphStore @runtime_checkable -class ConvertableGraphStore(Protocol, GraphStore): +class ConvertableGraphStore(Protocol): @classmethod def from_data( cls, @@ -63,7 +65,7 @@ def from_partition(cls, root: str, pid: int) -> GraphStore: # Adapted from LocalFeatureStore @runtime_checkable -class ConvertableFeatureStore(Protocol, FeatureStore): +class ConvertableFeatureStore(Protocol): @classmethod def from_data( cls, @@ -122,7 +124,7 @@ def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: self.path, pid) else: raise NotImplementedError - return (graph_store, feature_store) + return (feature_store, graph_store) # TODO: make profilable @@ -134,7 +136,7 @@ def create_remote_backend_from_triplets( node_method_to_call: str = "forward", edge_method_to_call: str | None = None, pre_transform: Callable[[TripletLike], TripletLike] | None = None, - path: str = '', n_parts: int = 1) -> RemoteGraphBackendLoader: + path: str = '', n_parts: int = 1, node_method_kwargs: Optional[Dict[str, Any]] = None, edge_method_kwargs: Optional[Dict[str, Any]] = None) -> RemoteGraphBackendLoader: # Will return attribute errors for missing attributes if not issubclass(graph_db, ConvertableGraphStore): @@ -147,24 +149,28 @@ def create_remote_backend_from_triplets( getattr(feature_db, "from_partition") # Resolve callable methods + node_method_kwargs = node_method_kwargs \ + if node_method_kwargs is not None else dict() + edge_embedding_model = edge_embedding_model \ if edge_embedding_model is not None else node_embedding_model edge_method_to_call = edge_method_to_call \ if edge_method_to_call is not None else node_method_to_call + edge_method_kwargs = edge_method_kwargs \ + if edge_method_kwargs is not None else node_method_kwargs # These will return AttributeErrors if they don't exist node_model = getattr(node_embedding_model, node_method_to_call) edge_model = getattr(edge_embedding_model, edge_method_to_call) - indexer = LargeGraphIndexer.from_triplets(triplets) + indexer = LargeGraphIndexer.from_triplets(triplets, pre_transform=pre_transform) - node_feats = node_model(indexer.get_node_features()) + node_feats = node_model(indexer.get_node_features(), **node_method_kwargs) indexer.add_node_feature('x', node_feats) edge_feats = edge_model( - indexer.get_edge_features(feature_name=EDGE_RELATION)) - indexer.add_edge_feature('edge_attr', edge_feats, - map_from_feature=EDGE_RELATION) + indexer.get_unique_edge_features(feature_name=EDGE_RELATION), **edge_method_kwargs) + indexer.add_edge_feature(new_feature_name="edge_attr", new_feature_vals=edge_feats, map_from_feature=EDGE_RELATION) data = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index f1fa23236381..bf3b5aec5fe8 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -1,4 +1,4 @@ -from typing import Iterable, Type +from typing import Iterable, Type, Optional, Dict, Any import torch from torch import Tensor @@ -11,11 +11,12 @@ # NOTE: Only compatible with Homogeneous graphs for now class KNNRAGFeatureStore(LocalFeatureStore): - def __init__(self, enc_model: Type[Module], *args, **kwargs): + def __init__(self, enc_model: Type[Module], model_kwargs: Optional[Dict[str, Any]] = None, *args, **kwargs): self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.enc_model = enc_model(*args, **kwargs).to(self.device) self.enc_model.eval() + self.model_kwargs = model_kwargs if model_kwargs is not None else dict() super().__init__() @property @@ -26,13 +27,9 @@ def x(self) -> Tensor: def edge_attr(self) -> Tensor: return self.get_tensor(group_name=(None, None), attr_name='edge_attr') - @property - def edge_id(self) -> Tensor: - return self.get_tensor(group_name=(None, None)) - def retrieve_seed_nodes(self, query: Iterable[str], - k_nodes: int = 5) -> Tensor: - query_enc = self.enc_model.encode(query) + k_nodes: int = 5) -> Iterable[Tensor]: + query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) topk = min(k_nodes, len(self.x)) topk_n_indices = [] @@ -40,19 +37,20 @@ def retrieve_seed_nodes(self, query: Iterable[str], _, indices = torch.topk(q, topk, largest=True) topk_n_indices.append(indices) - return torch.Tensor(topk_n_indices) + return topk_n_indices def retrieve_seed_edges(self, query: Iterable[str], - k_edges: int = 3) -> Tensor: - query_enc = self.enc_model.encode(query) + k_edges: int = 3) -> Iterable[Tensor]: + query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) prizes = pairwise_cosine_similarity(query_enc, self.edge_attr.to(self.device)) topk = min(k_edges, len(self.edge_attr)) - topk_n_indices = [] + topk_e_indices = [] for q in prizes: _, indices = torch.topk(q, topk, largest=True) - topk_n_indices.append(indices) - return torch.Tensor(topk_n_indices) + topk_e_indices.append(indices) + + return topk_e_indices class SentenceTransformerFeatureStore(KNNRAGFeatureStore): diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index 3c6aeb5f5105..40138a996953 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -1,27 +1,18 @@ -from abc import abstractmethod -from typing import Protocol, Tuple, Union +from typing import Tuple, Union, Optional import torch from torch import Tensor -from torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData +from torch_geometric.data import Data, FeatureStore, HeteroData from torch_geometric.distributed import LocalGraphStore -from torch_geometric.loader import NodeLoader -from torch_geometric.sampler import BaseSampler, NeighborSampler +from torch_geometric.loader import NodeLoader, NeighborLoader +from torch_geometric.sampler import NeighborSampler from torch_geometric.sampler.neighbor_sampler import NumNeighborsType -from torch_geometric.typing import InputEdges, InputNodes - - -class DataSampler(Protocol, BaseSampler): - @abstractmethod - def __init__( - self, data: Union[Data, HeteroData, Tuple[FeatureStore, - GraphStore]]) -> None: - ... +from torch_geometric.typing import InputEdges, InputNodes, EdgeTensorType class NeighborSamplingRAGGraphStore(LocalGraphStore): - def __init__(self, feature_store: FeatureStore, + def __init__(self, feature_store: Optional[FeatureStore] = None, num_neighbors: NumNeighborsType = [10], **kwargs): self.feature_store = feature_store self.num_neighbors = num_neighbors @@ -30,19 +21,31 @@ def __init__(self, feature_store: FeatureStore, super().__init__() def _init_sampler(self): + if self.feature_store is None: + raise AttributeError("Feature store not set yet.") self.sampler = NeighborSampler(data=(self.feature_store, self), num_neighbors=self.num_neighbors, **self.sample_kwargs) self._sampler_is_initialized = True + + def register_feature_store(self, feature_store: FeatureStore): + self.feature_store = feature_store def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: - ret = super().put_edge_id(edge_id, *args, **kwargs) + ret = super().put_edge_id(edge_id.contiguous(), *args, **kwargs) self._sampler_is_initialized = False return ret + + @property + def edge_index(self): + return self.get_edge_index(*self.edge_idx_args, **self.edge_idx_kwargs) - def put_edge_index(self, edge_index: Tuple[Tensor], *args, + def put_edge_index(self, edge_index: EdgeTensorType, *args, **kwargs) -> bool: ret = super().put_edge_index(edge_index, *args, **kwargs) + # HACK + self.edge_idx_args = args + self.edge_idx_kwargs = kwargs self._sampler_is_initialized = False return ret @@ -54,18 +57,25 @@ def retrieve_subgraph(self, seed_nodes: InputNodes, # Right now, only input nodes as tensors will be supported if not isinstance(seed_nodes, Tensor): raise NotImplementedError + device = seed_nodes.device if seed_edges is not None: if isinstance(seed_edges, Tensor): - seed_edges = seed_edges.reshape((-1)) + seed_edges = self.edge_index.to(device).T[seed_edges.to(device)].reshape((-1)) seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) else: raise NotImplementedError - seed_nodes = seed_nodes.unique() + seed_nodes = seed_nodes.unique().contiguous() num_nodes = len(seed_nodes) + loader = NeighborLoader(data=(self.feature_store, self), num_neighbors=[10], input_nodes=seed_nodes, batch_size=num_nodes) + # HACK: Fixes a bug where sampler columns aren't contiguous when initiated from local graph stores + loader.node_sampler.colptr = loader.node_sampler.colptr.contiguous() + loader.node_sampler.row = loader.node_sampler.row.contiguous() + ''' loader = NodeLoader(data=(self.feature_store, self), node_sampler=self.sampler, input_nodes=seed_nodes, batch_size=num_nodes) + ''' return next(iter(loader)) diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py index dcc5b2bdf68c..a78e704f3844 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/examples/llm_plus_gnn/rag_loader.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Callable, Optional, Protocol, Tuple, Union +from typing import Any, Callable, Optional, Protocol, Tuple, Union, runtime_checkable, Iterable import torch from torch import Tensor @@ -8,10 +8,11 @@ from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes -class RAGFeatureStore(Protocol, FeatureStore): +@runtime_checkable +class RAGFeatureStore(Protocol): """Feature store for remote GNN RAG backend.""" @abstractmethod - def retrieve_seed_nodes(self, query: Any, **kwargs) -> Tensor: + def retrieve_seed_nodes(self, query: Iterable[Any], **kwargs) -> Iterable[Tensor]: """Makes a comparison between the query and all the nodes to get all the closest nodes. Return the indices of the nodes that are to be seeds for the RAG Sampler. @@ -19,7 +20,7 @@ def retrieve_seed_nodes(self, query: Any, **kwargs) -> Tensor: ... @abstractmethod - def retieve_seed_edges(self, query: Any, **kwargs) -> EdgeTensorType: + def retrieve_seed_edges(self, query: Iterable[Any], **kwargs) -> Iterable[Tensor]: """Makes a comparison between the query and all the edges to get all the closest nodes. Returns the edge indices that are to be the seeds for the RAG Sampler. @@ -27,32 +28,40 @@ def retieve_seed_edges(self, query: Any, **kwargs) -> EdgeTensorType: ... -class RAGGraphStore(Protocol, GraphStore): +@runtime_checkable +class RAGGraphStore(Protocol): """Graph store for remote GNN RAG backend.""" @abstractmethod def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: """Sample a subgraph using the seeded nodes and edges.""" ... + + @abstractmethod + def register_feature_store(self, feature_store: FeatureStore): + """Register a feature store to be used with the sampler.""" + ... # TODO: Make compatible with Heterographs - class RagQueryLoader: - def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], + def __init__(self, data: Tuple[FeatureStore, GraphStore], local_filter: Optional[Callable] = None, **kwargs): fstore, gstore = data - self.feature_store: RAGFeatureStore = fstore - self.graph_store: RAGGraphStore = gstore + assert issubclass(type(fstore), RAGFeatureStore) + assert issubclass(type(gstore), RAGGraphStore) + self.feature_store: FeatureStore = fstore + self.graph_store: GraphStore = gstore + self.graph_store.register_feature_store(self.feature_store) self.local_filter = local_filter def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature attributes. """ - seed_nodes = self.feature_store.retrieve_seed_nodes(query) - seed_edges = self.feature_store.retrieve_seed_edges(query) + seed_nodes = self.feature_store.retrieve_seed_nodes([query])[0] + seed_edges = self.feature_store.retrieve_seed_edges([query])[0] subgraph = self.graph_store.retrieve_subgraph(seed_nodes, seed_edges).edge_index diff --git a/examples/llm_plus_gnn/test_rag.ipynb b/examples/llm_plus_gnn/test_rag.ipynb new file mode 100644 index 000000000000..837231c38465 --- /dev/null +++ b/examples/llm_plus_gnn/test_rag.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from profiling_utils import create_remote_backend_from_triplets\n", + "from rag_feature_store import SentenceTransformerFeatureStore\n", + "from rag_graph_store import NeighborSamplingRAGGraphStore\n", + "from rag_loader import RagQueryLoader\n", + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet\n", + "from itertools import chain\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading graph...\n", + "Encoding questions...\n", + "Retrieving subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 59.31it/s]00:00 1\u001b[0m subg \u001b[38;5;241m=\u001b[39m \u001b[43mquery_loader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/rag_loader.py:63\u001b[0m, in \u001b[0;36mRagQueryLoader.query\u001b[0;34m(self, query)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mquery\u001b[39m(\u001b[38;5;28mself\u001b[39m, query: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Data:\n\u001b[1;32m 60\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Retrieve a subgraph associated with the query with all its feature\u001b[39;00m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;124;03m attributes.\u001b[39;00m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 63\u001b[0m seed_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeature_store\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve_seed_nodes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 64\u001b[0m seed_edges \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfeature_store\u001b[38;5;241m.\u001b[39mretrieve_seed_edges(query)\n\u001b[1;32m 66\u001b[0m subgraph \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_store\u001b[38;5;241m.\u001b[39mretrieve_subgraph(seed_nodes,\n\u001b[1;32m 67\u001b[0m seed_edges)\u001b[38;5;241m.\u001b[39medge_index\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/rag_feature_store.py:44\u001b[0m, in \u001b[0;36mKNNRAGFeatureStore.retrieve_seed_nodes\u001b[0;34m(self, query, k_nodes)\u001b[0m\n\u001b[1;32m 41\u001b[0m _, indices \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtopk(q, topk, largest\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 42\u001b[0m topk_n_indices\u001b[38;5;241m.\u001b[39mappend(indices)\n\u001b[0;32m---> 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtopk_n_indices\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mValueError\u001b[0m: only one element tensors can be converted to Python scalars" + ] + } + ], + "source": [ + "subg = query_loader.query(query)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index cf1d777a57c6..59c2f062bbc1 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -283,7 +283,7 @@ def add_edge_feature( feature_keys = self.get_unique_edge_features(map_from_feature) if len(feature_keys) != len(new_feature_vals): raise AttributeError( - "Expected encodings for {len(feature_keys)} unique features, " + f"Expected encodings for {len(feature_keys)} unique features, " + f"but got {len(new_feature_vals)} encodings.") if map_from_feature == EDGE_PID: From 0ac325895ebccb5b05afc4e917352767484dfd72 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 11 Jun 2024 19:23:57 -0700 Subject: [PATCH 618/752] pre commit --- examples/llm_plus_gnn/profiling_utils.py | 31 +++++++++++++--------- examples/llm_plus_gnn/rag_feature_store.py | 17 +++++++----- examples/llm_plus_gnn/rag_graph_store.py | 20 ++++++++------ examples/llm_plus_gnn/rag_loader.py | 22 +++++++++++---- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py index 9b6b54b0055e..ba0b96f21979 100644 --- a/examples/llm_plus_gnn/profiling_utils.py +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from enum import Enum, auto from typing import ( + Any, Callable, Dict, Iterable, @@ -9,7 +10,6 @@ Tuple, Type, runtime_checkable, - Any, ) import torch @@ -129,14 +129,17 @@ def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: # TODO: make profilable def create_remote_backend_from_triplets( - triplets: Iterable[TripletLike], node_embedding_model: Module, - edge_embedding_model: Module | None = None, - graph_db: Type[ConvertableGraphStore] = LocalGraphStore, - feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, - node_method_to_call: str = "forward", - edge_method_to_call: str | None = None, - pre_transform: Callable[[TripletLike], TripletLike] | None = None, - path: str = '', n_parts: int = 1, node_method_kwargs: Optional[Dict[str, Any]] = None, edge_method_kwargs: Optional[Dict[str, Any]] = None) -> RemoteGraphBackendLoader: + triplets: Iterable[TripletLike], node_embedding_model: Module, + edge_embedding_model: Module | None = None, + graph_db: Type[ConvertableGraphStore] = LocalGraphStore, + feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, + node_method_to_call: str = "forward", + edge_method_to_call: str | None = None, + pre_transform: Callable[[TripletLike], TripletLike] | None = None, + path: str = '', n_parts: int = 1, + node_method_kwargs: Optional[Dict[str, Any]] = None, + edge_method_kwargs: Optional[Dict[str, Any]] = None +) -> RemoteGraphBackendLoader: # Will return attribute errors for missing attributes if not issubclass(graph_db, ConvertableGraphStore): @@ -163,14 +166,18 @@ def create_remote_backend_from_triplets( node_model = getattr(node_embedding_model, node_method_to_call) edge_model = getattr(edge_embedding_model, edge_method_to_call) - indexer = LargeGraphIndexer.from_triplets(triplets, pre_transform=pre_transform) + indexer = LargeGraphIndexer.from_triplets(triplets, + pre_transform=pre_transform) node_feats = node_model(indexer.get_node_features(), **node_method_kwargs) indexer.add_node_feature('x', node_feats) edge_feats = edge_model( - indexer.get_unique_edge_features(feature_name=EDGE_RELATION), **edge_method_kwargs) - indexer.add_edge_feature(new_feature_name="edge_attr", new_feature_vals=edge_feats, map_from_feature=EDGE_RELATION) + indexer.get_unique_edge_features(feature_name=EDGE_RELATION), + **edge_method_kwargs) + indexer.add_edge_feature(new_feature_name="edge_attr", + new_feature_vals=edge_feats, + map_from_feature=EDGE_RELATION) data = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index bf3b5aec5fe8..e469ce3478ff 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -1,4 +1,4 @@ -from typing import Iterable, Type, Optional, Dict, Any +from typing import Any, Dict, Iterable, Optional, Type import torch from torch import Tensor @@ -11,12 +11,15 @@ # NOTE: Only compatible with Homogeneous graphs for now class KNNRAGFeatureStore(LocalFeatureStore): - def __init__(self, enc_model: Type[Module], model_kwargs: Optional[Dict[str, Any]] = None, *args, **kwargs): + def __init__(self, enc_model: Type[Module], + model_kwargs: Optional[Dict[str, + Any]] = None, *args, **kwargs): self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.enc_model = enc_model(*args, **kwargs).to(self.device) self.enc_model.eval() - self.model_kwargs = model_kwargs if model_kwargs is not None else dict() + self.model_kwargs = model_kwargs if model_kwargs is not None else dict( + ) super().__init__() @property @@ -29,7 +32,8 @@ def edge_attr(self) -> Tensor: def retrieve_seed_nodes(self, query: Iterable[str], k_nodes: int = 5) -> Iterable[Tensor]: - query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) + query_enc = self.enc_model.encode(query, + **self.model_kwargs).to(self.device) prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) topk = min(k_nodes, len(self.x)) topk_n_indices = [] @@ -41,14 +45,15 @@ def retrieve_seed_nodes(self, query: Iterable[str], def retrieve_seed_edges(self, query: Iterable[str], k_edges: int = 3) -> Iterable[Tensor]: - query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) + query_enc = self.enc_model.encode(query, + **self.model_kwargs).to(self.device) prizes = pairwise_cosine_similarity(query_enc, self.edge_attr.to(self.device)) topk = min(k_edges, len(self.edge_attr)) topk_e_indices = [] for q in prizes: _, indices = torch.topk(q, topk, largest=True) - topk_e_indices.append(indices) + topk_e_indices.append(indices) return topk_e_indices diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index 40138a996953..c9ecafa4accc 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -1,14 +1,14 @@ -from typing import Tuple, Union, Optional +from typing import Optional, Union import torch from torch import Tensor from torch_geometric.data import Data, FeatureStore, HeteroData from torch_geometric.distributed import LocalGraphStore -from torch_geometric.loader import NodeLoader, NeighborLoader +from torch_geometric.loader import NeighborLoader, NodeLoader from torch_geometric.sampler import NeighborSampler from torch_geometric.sampler.neighbor_sampler import NumNeighborsType -from torch_geometric.typing import InputEdges, InputNodes, EdgeTensorType +from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes class NeighborSamplingRAGGraphStore(LocalGraphStore): @@ -27,7 +27,7 @@ def _init_sampler(self): num_neighbors=self.num_neighbors, **self.sample_kwargs) self._sampler_is_initialized = True - + def register_feature_store(self, feature_store: FeatureStore): self.feature_store = feature_store @@ -35,7 +35,7 @@ def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: ret = super().put_edge_id(edge_id.contiguous(), *args, **kwargs) self._sampler_is_initialized = False return ret - + @property def edge_index(self): return self.get_edge_index(*self.edge_idx_args, **self.edge_idx_kwargs) @@ -61,15 +61,19 @@ def retrieve_subgraph(self, seed_nodes: InputNodes, if seed_edges is not None: if isinstance(seed_edges, Tensor): - seed_edges = self.edge_index.to(device).T[seed_edges.to(device)].reshape((-1)) + seed_edges = self.edge_index.to(device).T[seed_edges.to( + device)].reshape((-1)) seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) else: raise NotImplementedError seed_nodes = seed_nodes.unique().contiguous() num_nodes = len(seed_nodes) - loader = NeighborLoader(data=(self.feature_store, self), num_neighbors=[10], input_nodes=seed_nodes, batch_size=num_nodes) - # HACK: Fixes a bug where sampler columns aren't contiguous when initiated from local graph stores + loader = NeighborLoader(data=(self.feature_store, self), + num_neighbors=[10], input_nodes=seed_nodes, + batch_size=num_nodes) + # HACK: Fixes a bug where sampler columns aren't contiguous when + # initiated from local graph stores loader.node_sampler.colptr = loader.node_sampler.colptr.contiguous() loader.node_sampler.row = loader.node_sampler.row.contiguous() ''' diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py index a78e704f3844..305201060422 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/examples/llm_plus_gnn/rag_loader.py @@ -1,18 +1,28 @@ from abc import abstractmethod -from typing import Any, Callable, Optional, Protocol, Tuple, Union, runtime_checkable, Iterable +from typing import ( + Any, + Callable, + Iterable, + Optional, + Protocol, + Tuple, + Union, + runtime_checkable, +) import torch from torch import Tensor from torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData -from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes +from torch_geometric.typing import InputEdges, InputNodes @runtime_checkable class RAGFeatureStore(Protocol): """Feature store for remote GNN RAG backend.""" @abstractmethod - def retrieve_seed_nodes(self, query: Iterable[Any], **kwargs) -> Iterable[Tensor]: + def retrieve_seed_nodes(self, query: Iterable[Any], + **kwargs) -> Iterable[Tensor]: """Makes a comparison between the query and all the nodes to get all the closest nodes. Return the indices of the nodes that are to be seeds for the RAG Sampler. @@ -20,7 +30,8 @@ def retrieve_seed_nodes(self, query: Iterable[Any], **kwargs) -> Iterable[Tensor ... @abstractmethod - def retrieve_seed_edges(self, query: Iterable[Any], **kwargs) -> Iterable[Tensor]: + def retrieve_seed_edges(self, query: Iterable[Any], + **kwargs) -> Iterable[Tensor]: """Makes a comparison between the query and all the edges to get all the closest nodes. Returns the edge indices that are to be the seeds for the RAG Sampler. @@ -36,7 +47,7 @@ def retrieve_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges) -> Union[Data, HeteroData]: """Sample a subgraph using the seeded nodes and edges.""" ... - + @abstractmethod def register_feature_store(self, feature_store: FeatureStore): """Register a feature store to be used with the sampler.""" @@ -45,6 +56,7 @@ def register_feature_store(self, feature_store: FeatureStore): # TODO: Make compatible with Heterographs + class RagQueryLoader: def __init__(self, data: Tuple[FeatureStore, GraphStore], local_filter: Optional[Callable] = None, **kwargs): From e6b2cd0fc5f5c9d288c9286126baecf639fc2821 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 12 Jun 2024 18:21:29 -0700 Subject: [PATCH 619/752] debugging done, begin qol cleanup and writing tests --- examples/llm_plus_gnn/profiling_utils.py | 2 +- examples/llm_plus_gnn/rag_feature_store.py | 55 ++++++++--- examples/llm_plus_gnn/rag_graph_store.py | 46 ++++----- examples/llm_plus_gnn/rag_loader.py | 67 ++++++------- examples/llm_plus_gnn/test_rag.ipynb | 110 +++++++++++++++++---- 5 files changed, 186 insertions(+), 94 deletions(-) diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py index ba0b96f21979..2bc537f08d4a 100644 --- a/examples/llm_plus_gnn/profiling_utils.py +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -110,7 +110,7 @@ def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: data_obj = torch.load(self.path) graph_store = self.graph_store_type.from_data( edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, - num_nodes=data_obj.num_nodes, is_sorted=True) + num_nodes=data_obj.num_nodes) feature_store = self.feature_store_type.from_data( node_id=data_obj['node_id'], x=data_obj.x, edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index e469ce3478ff..0ca4c3de81ed 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -1,12 +1,16 @@ -from typing import Any, Dict, Iterable, Optional, Type +from collections.abc import Iterable, Iterator +from typing import Any, Dict, Optional, Type, Union import torch from torch import Tensor from torch.nn import Module from torchmetrics.functional import pairwise_cosine_similarity +from torch_geometric.data import Data, HeteroData from torch_geometric.distributed import LocalFeatureStore from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput +from torch_geometric.typing import InputEdges, InputNodes # NOTE: Only compatible with Homogeneous graphs for now @@ -18,8 +22,8 @@ def __init__(self, enc_model: Type[Module], "cuda" if torch.cuda.is_available() else "cpu") self.enc_model = enc_model(*args, **kwargs).to(self.device) self.enc_model.eval() - self.model_kwargs = model_kwargs if model_kwargs is not None else dict( - ) + self.model_kwargs = \ + model_kwargs if model_kwargs is not None else dict() super().__init__() @property @@ -30,32 +34,57 @@ def x(self) -> Tensor: def edge_attr(self) -> Tensor: return self.get_tensor(group_name=(None, None), attr_name='edge_attr') - def retrieve_seed_nodes(self, query: Iterable[str], - k_nodes: int = 5) -> Iterable[Tensor]: + def retrieve_seed_nodes(self, query: Any, k_nodes: int = 5) -> InputNodes: + return next(self._retrieve_seed_nodes_batch([query], k_nodes)) + + def _retrieve_seed_nodes_batch(self, query: Iterable[Any], + k_nodes: int) -> Iterator[InputNodes]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError + query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) topk = min(k_nodes, len(self.x)) - topk_n_indices = [] for q in prizes: _, indices = torch.topk(q, topk, largest=True) - topk_n_indices.append(indices) + yield indices - return topk_n_indices + def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: + return next(self._retrieve_seed_edges_batch([query], k_edges)) + + def _retrieve_seed_edges_batch(self, query: Iterable[Any], + k_edges: int) -> Iterator[InputEdges]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError - def retrieve_seed_edges(self, query: Iterable[str], - k_edges: int = 3) -> Iterable[Tensor]: query_enc = self.enc_model.encode(query, **self.model_kwargs).to(self.device) + prizes = pairwise_cosine_similarity(query_enc, self.edge_attr.to(self.device)) topk = min(k_edges, len(self.edge_attr)) - topk_e_indices = [] for q in prizes: _, indices = torch.topk(q, topk, largest=True) - topk_e_indices.append(indices) + yield indices + + def load_subgraph( + self, sample: Union[SamplerOutput, HeteroSamplerOutput] + ) -> Union[Data, HeteroData]: + + if isinstance(sample, HeteroSamplerOutput): + raise NotImplementedError + + # NOTE: torch_geometric.loader.utils.filter_custom_store can be used + # here if it supported edge features + node_id = sample.node + edge_id = sample.edge + edge_index = torch.stack((sample.row, sample.col), dim=0) + x = self.x[node_id] + edge_attr = self.edge_attr[edge_id] - return topk_e_indices + return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + node_idx=node_id, edge_idx=edge_id) class SentenceTransformerFeatureStore(KNNRAGFeatureStore): diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index c9ecafa4accc..2e2d0e147f88 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -3,17 +3,22 @@ import torch from torch import Tensor -from torch_geometric.data import Data, FeatureStore, HeteroData +from torch_geometric.data import FeatureStore from torch_geometric.distributed import LocalGraphStore -from torch_geometric.loader import NeighborLoader, NodeLoader -from torch_geometric.sampler import NeighborSampler +from torch_geometric.sampler import ( + HeteroSamplerOutput, + NeighborSampler, + NodeSamplerInput, + SamplerOutput, +) from torch_geometric.sampler.neighbor_sampler import NumNeighborsType from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes class NeighborSamplingRAGGraphStore(LocalGraphStore): def __init__(self, feature_store: Optional[FeatureStore] = None, - num_neighbors: NumNeighborsType = [10], **kwargs): + num_neighbors: NumNeighborsType = [100, 100, 100, 100, + 100], **kwargs): self.feature_store = feature_store self.num_neighbors = num_neighbors self.sample_kwargs = kwargs @@ -22,7 +27,7 @@ def __init__(self, feature_store: Optional[FeatureStore] = None, def _init_sampler(self): if self.feature_store is None: - raise AttributeError("Feature store not set yet.") + raise AttributeError("Feature store not registered yet.") self.sampler = NeighborSampler(data=(self.feature_store, self), num_neighbors=self.num_neighbors, **self.sample_kwargs) @@ -49,37 +54,34 @@ def put_edge_index(self, edge_index: EdgeTensorType, *args, self._sampler_is_initialized = False return ret - def retrieve_subgraph(self, seed_nodes: InputNodes, - seed_edges: InputEdges) -> Union[Data, HeteroData]: + def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, + *args, + **kwargs) -> Union[SamplerOutput, HeteroSamplerOutput]: if not self._sampler_is_initialized: self._init_sampler() - # Right now, only input nodes as tensors will be supported + # FIXME: Right now, only input nodes/edges as tensors are be supported if not isinstance(seed_nodes, Tensor): raise NotImplementedError + if not isinstance(seed_edges, Tensor): + raise NotImplementedError device = seed_nodes.device - if seed_edges is not None: - if isinstance(seed_edges, Tensor): - seed_edges = self.edge_index.to(device).T[seed_edges.to( - device)].reshape((-1)) - seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) - else: - raise NotImplementedError + # TODO: Call sample_from_edges for seed_edges + seed_edges = self.edge_index.to(device).T[seed_edges.to( + device)].reshape((-1)) + seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) seed_nodes = seed_nodes.unique().contiguous() - num_nodes = len(seed_nodes) + ''' loader = NeighborLoader(data=(self.feature_store, self), num_neighbors=[10], input_nodes=seed_nodes, batch_size=num_nodes) - # HACK: Fixes a bug where sampler columns aren't contiguous when - # initiated from local graph stores - loader.node_sampler.colptr = loader.node_sampler.colptr.contiguous() - loader.node_sampler.row = loader.node_sampler.row.contiguous() - ''' loader = NodeLoader(data=(self.feature_store, self), node_sampler=self.sampler, input_nodes=seed_nodes, batch_size=num_nodes) ''' + node_sample_input = NodeSamplerInput(input_id=None, node=seed_nodes) + out = self.sampler.sample_from_nodes(node_sample_input) - return next(iter(loader)) + return out diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py index 305201060422..cada1a7aa70b 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/examples/llm_plus_gnn/rag_loader.py @@ -1,28 +1,15 @@ from abc import abstractmethod -from typing import ( - Any, - Callable, - Iterable, - Optional, - Protocol, - Tuple, - Union, - runtime_checkable, -) - -import torch -from torch import Tensor - -from torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData +from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union + +from torch_geometric.data import Data, FeatureStore, HeteroData +from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput from torch_geometric.typing import InputEdges, InputNodes -@runtime_checkable class RAGFeatureStore(Protocol): """Feature store for remote GNN RAG backend.""" @abstractmethod - def retrieve_seed_nodes(self, query: Iterable[Any], - **kwargs) -> Iterable[Tensor]: + def retrieve_seed_nodes(self, query: Any, **kwargs) -> InputNodes: """Makes a comparison between the query and all the nodes to get all the closest nodes. Return the indices of the nodes that are to be seeds for the RAG Sampler. @@ -30,27 +17,34 @@ def retrieve_seed_nodes(self, query: Iterable[Any], ... @abstractmethod - def retrieve_seed_edges(self, query: Iterable[Any], - **kwargs) -> Iterable[Tensor]: + def retrieve_seed_edges(self, query: Any, **kwargs) -> InputEdges: """Makes a comparison between the query and all the edges to get all the closest nodes. Returns the edge indices that are to be the seeds for the RAG Sampler. """ ... + @abstractmethod + def load_subgraph( + self, sample: Union[SamplerOutput, HeteroSamplerOutput] + ) -> Union[Data, HeteroData]: + """Combines sampled subgraph output with features in a Data object.""" + ... + -@runtime_checkable class RAGGraphStore(Protocol): """Graph store for remote GNN RAG backend.""" @abstractmethod - def retrieve_subgraph(self, seed_nodes: InputNodes, - seed_edges: InputEdges) -> Union[Data, HeteroData]: + def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, + **kwargs) -> Union[SamplerOutput, HeteroSamplerOutput]: """Sample a subgraph using the seeded nodes and edges.""" ... @abstractmethod def register_feature_store(self, feature_store: FeatureStore): - """Register a feature store to be used with the sampler.""" + """Register a feature store to be used with the sampler. Samplers need + info from the feature store in order to work properly on HeteroGraphs. + """ ... @@ -58,13 +52,13 @@ def register_feature_store(self, feature_store: FeatureStore): class RagQueryLoader: - def __init__(self, data: Tuple[FeatureStore, GraphStore], - local_filter: Optional[Callable] = None, **kwargs): + def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], + local_filter: Optional[Callable] = None, + sampler_kwargs: Optional[Dict[str, Any]] = None, + loader_kwargs: Optional[Dict[str, Any]] = None): fstore, gstore = data - assert issubclass(type(fstore), RAGFeatureStore) - assert issubclass(type(gstore), RAGGraphStore) - self.feature_store: FeatureStore = fstore - self.graph_store: GraphStore = gstore + self.feature_store = fstore + self.graph_store = gstore self.graph_store.register_feature_store(self.feature_store) self.local_filter = local_filter @@ -72,17 +66,14 @@ def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature attributes. """ - seed_nodes = self.feature_store.retrieve_seed_nodes([query])[0] - seed_edges = self.feature_store.retrieve_seed_edges([query])[0] + seed_nodes = self.feature_store.retrieve_seed_nodes(query) + seed_edges = self.feature_store.retrieve_seed_edges(query) - subgraph = self.graph_store.retrieve_subgraph(seed_nodes, - seed_edges).edge_index + subgraph_sample = self.graph_store.sample_subgraph( + seed_nodes, seed_edges) - nodes = torch.cat((subgraph[0], subgraph[1]), dim=0).unique() - x = self.feature_store.multi_get_tensor(nodes) - edge_attr = self.feature_store.multi_get_tensor(subgraph) + data = self.feature_store.load_subgraph(sample=subgraph_sample) - data = Data(x=x, edge_attr=edge_attr, edge_index=subgraph) if self.local_filter: data = self.local_filter(data) return data diff --git a/examples/llm_plus_gnn/test_rag.ipynb b/examples/llm_plus_gnn/test_rag.ipynb index 837231c38465..922d81198de8 100644 --- a/examples/llm_plus_gnn/test_rag.ipynb +++ b/examples/llm_plus_gnn/test_rag.ipynb @@ -22,6 +22,7 @@ "from torch_geometric.datasets import UpdatedWebQSPDataset\n", "from torch_geometric.nn.nlp import SentenceTransformer\n", "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet\n", + "from torch_geometric.data import get_features_for_triplets_groups, Data\n", "from itertools import chain\n", "import torch" ] @@ -51,8 +52,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "10it [00:00, 59.31it/s]00:00 1\u001b[0m subg \u001b[38;5;241m=\u001b[39m \u001b[43mquery_loader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/rag_loader.py:63\u001b[0m, in \u001b[0;36mRagQueryLoader.query\u001b[0;34m(self, query)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mquery\u001b[39m(\u001b[38;5;28mself\u001b[39m, query: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Data:\n\u001b[1;32m 60\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Retrieve a subgraph associated with the query with all its feature\u001b[39;00m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;124;03m attributes.\u001b[39;00m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 63\u001b[0m seed_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeature_store\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve_seed_nodes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 64\u001b[0m seed_edges \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfeature_store\u001b[38;5;241m.\u001b[39mretrieve_seed_edges(query)\n\u001b[1;32m 66\u001b[0m subgraph \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_store\u001b[38;5;241m.\u001b[39mretrieve_subgraph(seed_nodes,\n\u001b[1;32m 67\u001b[0m seed_edges)\u001b[38;5;241m.\u001b[39medge_index\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/rag_feature_store.py:44\u001b[0m, in \u001b[0;36mKNNRAGFeatureStore.retrieve_seed_nodes\u001b[0;34m(self, query, k_nodes)\u001b[0m\n\u001b[1;32m 41\u001b[0m _, indices \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtopk(q, topk, largest\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 42\u001b[0m topk_n_indices\u001b[38;5;241m.\u001b[39mappend(indices)\n\u001b[0;32m---> 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtopk_n_indices\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[0;31mValueError\u001b[0m: only one element tensors can be converted to Python scalars" + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 59.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.47500327696945865 0.22471144012078118 0.5290677674578603\n", + "0.6829990824485516 0.10915320606950563 0.2714877039201363\n", + "0.4894219425874951 0.05582922824302135 0.5027726432532348\n", + "0.5146939310525626 0.14810335349092907 0.47180385288966725\n", + "0.48978896316686327 0.038508934072704865 0.5047106325706595\n", + "0.556979944946913 0.07315452458620272 0.4376581134803036\n", + "0.4486826582776249 0.1033375226923614 0.5568096313017307\n", + "0.6349980338183248 0.06187745246000604 0.35482475118996104\n", + "0.6690260846768908 0.1052010529182173 0.29521141110545085\n", + "0.5178660374885307 0.16869006042463552 0.47266231748990367\n" ] } ], "source": [ - "subg = query_loader.query(query)" + "for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs):\n", + " print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From a782e028fd51b3359fafc6de8de54d37bc9994e0 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 13 Jun 2024 18:18:51 -0700 Subject: [PATCH 620/752] fstore and gstore configurable from loader --- examples/llm_plus_gnn/rag_graph_store.py | 32 +++++++++++++----------- examples/llm_plus_gnn/rag_loader.py | 6 +++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index 2e2d0e147f88..60128ca1ba01 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -17,10 +17,9 @@ class NeighborSamplingRAGGraphStore(LocalGraphStore): def __init__(self, feature_store: Optional[FeatureStore] = None, - num_neighbors: NumNeighborsType = [100, 100, 100, 100, - 100], **kwargs): + num_neighbors: NumNeighborsType = [1], **kwargs): self.feature_store = feature_store - self.num_neighbors = num_neighbors + self._num_neighbors = num_neighbors self.sample_kwargs = kwargs self._sampler_is_initialized = False super().__init__() @@ -29,12 +28,13 @@ def _init_sampler(self): if self.feature_store is None: raise AttributeError("Feature store not registered yet.") self.sampler = NeighborSampler(data=(self.feature_store, self), - num_neighbors=self.num_neighbors, + num_neighbors=self._num_neighbors, **self.sample_kwargs) self._sampler_is_initialized = True def register_feature_store(self, feature_store: FeatureStore): self.feature_store = feature_store + self._sampler_is_initialized = False def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: ret = super().put_edge_id(edge_id.contiguous(), *args, **kwargs) @@ -53,12 +53,22 @@ def put_edge_index(self, edge_index: EdgeTensorType, *args, self.edge_idx_kwargs = kwargs self._sampler_is_initialized = False return ret + + @property + def num_neighbors(self): + return self._num_neighbors + + @num_neighbors.setter + def num_neighbors(self, num_neighbors: NumNeighborsType): + self._num_neighbors = num_neighbors + if hasattr(self, 'sampler'): + self.sampler.num_neighbors = num_neighbors - def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, - *args, - **kwargs) -> Union[SamplerOutput, HeteroSamplerOutput]: + def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, num_neighbors: Optional[NumNeighborsType] = None) -> Union[SamplerOutput, HeteroSamplerOutput]: if not self._sampler_is_initialized: self._init_sampler() + if num_neighbors is not None: + self.num_neighbors = num_neighbors # FIXME: Right now, only input nodes/edges as tensors are be supported if not isinstance(seed_nodes, Tensor): @@ -73,14 +83,6 @@ def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) seed_nodes = seed_nodes.unique().contiguous() - ''' - loader = NeighborLoader(data=(self.feature_store, self), - num_neighbors=[10], input_nodes=seed_nodes, - batch_size=num_nodes) - loader = NodeLoader(data=(self.feature_store, self), - node_sampler=self.sampler, input_nodes=seed_nodes, - batch_size=num_nodes) - ''' node_sample_input = NodeSamplerInput(input_id=None, node=seed_nodes) out = self.sampler.sample_from_nodes(node_sample_input) diff --git a/examples/llm_plus_gnn/rag_loader.py b/examples/llm_plus_gnn/rag_loader.py index cada1a7aa70b..c8722e9c3e41 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/examples/llm_plus_gnn/rag_loader.py @@ -61,6 +61,8 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], self.graph_store = gstore self.graph_store.register_feature_store(self.feature_store) self.local_filter = local_filter + self.sampler_kwargs = sampler_kwargs or {} + self.loader_kwargs = loader_kwargs or {} def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature @@ -70,9 +72,9 @@ def query(self, query: Any) -> Data: seed_edges = self.feature_store.retrieve_seed_edges(query) subgraph_sample = self.graph_store.sample_subgraph( - seed_nodes, seed_edges) + seed_nodes, seed_edges, **self.sampler_kwargs) - data = self.feature_store.load_subgraph(sample=subgraph_sample) + data = self.feature_store.load_subgraph(sample=subgraph_sample, **self.loader_kwargs) if self.local_filter: data = self.local_filter(data) From 0e580801cc979a8b503226b69f6e5871bbee3f5f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 13 Jun 2024 18:29:51 -0700 Subject: [PATCH 621/752] move rag loader --- torch_geometric/loader/__init__.py | 2 ++ {examples/llm_plus_gnn => torch_geometric/loader}/rag_loader.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) rename {examples/llm_plus_gnn => torch_geometric/loader}/rag_loader.py (99%) diff --git a/torch_geometric/loader/__init__.py b/torch_geometric/loader/__init__.py index 266f498a113b..7e83c35befb6 100644 --- a/torch_geometric/loader/__init__.py +++ b/torch_geometric/loader/__init__.py @@ -22,6 +22,7 @@ from .prefetch import PrefetchLoader from .cache import CachedLoader from .mixin import AffinityMixin +from .rag_loader import RAGQueryLoader __all__ = classes = [ 'DataLoader', @@ -50,6 +51,7 @@ 'PrefetchLoader', 'CachedLoader', 'AffinityMixin', + 'RAGQueryLoader', ] RandomNodeSampler = deprecated( diff --git a/examples/llm_plus_gnn/rag_loader.py b/torch_geometric/loader/rag_loader.py similarity index 99% rename from examples/llm_plus_gnn/rag_loader.py rename to torch_geometric/loader/rag_loader.py index c8722e9c3e41..41ef37a8c014 100644 --- a/examples/llm_plus_gnn/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -51,7 +51,7 @@ def register_feature_store(self, feature_store: FeatureStore): # TODO: Make compatible with Heterographs -class RagQueryLoader: +class RAGQueryLoader: def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], local_filter: Optional[Callable] = None, sampler_kwargs: Optional[Dict[str, Any]] = None, From 8fd61f4e99cd41d62220e0de4ebc5f4bef9bcbe8 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 13 Jun 2024 18:30:22 -0700 Subject: [PATCH 622/752] pre commit --- examples/llm_plus_gnn/rag_graph_store.py | 9 ++++++--- torch_geometric/loader/rag_loader.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index 60128ca1ba01..ee780aff0860 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -53,18 +53,21 @@ def put_edge_index(self, edge_index: EdgeTensorType, *args, self.edge_idx_kwargs = kwargs self._sampler_is_initialized = False return ret - + @property def num_neighbors(self): return self._num_neighbors - + @num_neighbors.setter def num_neighbors(self, num_neighbors: NumNeighborsType): self._num_neighbors = num_neighbors if hasattr(self, 'sampler'): self.sampler.num_neighbors = num_neighbors - def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, num_neighbors: Optional[NumNeighborsType] = None) -> Union[SamplerOutput, HeteroSamplerOutput]: + def sample_subgraph( + self, seed_nodes: InputNodes, seed_edges: InputEdges, + num_neighbors: Optional[NumNeighborsType] = None + ) -> Union[SamplerOutput, HeteroSamplerOutput]: if not self._sampler_is_initialized: self._init_sampler() if num_neighbors is not None: diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 41ef37a8c014..6615568d2151 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -74,7 +74,8 @@ def query(self, query: Any) -> Data: subgraph_sample = self.graph_store.sample_subgraph( seed_nodes, seed_edges, **self.sampler_kwargs) - data = self.feature_store.load_subgraph(sample=subgraph_sample, **self.loader_kwargs) + data = self.feature_store.load_subgraph(sample=subgraph_sample, + **self.loader_kwargs) if self.local_filter: data = self.local_filter(data) From 7f001c811547cdf98f252fd0597467bf133866bd Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 17 Jun 2024 14:04:19 -0700 Subject: [PATCH 623/752] pre transform working --- examples/llm_plus_gnn/test_rag.ipynb | 114 ++++++++++++++---- .../datasets/updated_web_qsp_dataset.py | 7 ++ torch_geometric/loader/rag_loader.py | 14 ++- 3 files changed, 110 insertions(+), 25 deletions(-) diff --git a/examples/llm_plus_gnn/test_rag.ipynb b/examples/llm_plus_gnn/test_rag.ipynb index 922d81198de8..286f7b495499 100644 --- a/examples/llm_plus_gnn/test_rag.ipynb +++ b/examples/llm_plus_gnn/test_rag.ipynb @@ -18,13 +18,14 @@ "from profiling_utils import create_remote_backend_from_triplets\n", "from rag_feature_store import SentenceTransformerFeatureStore\n", "from rag_graph_store import NeighborSamplingRAGGraphStore\n", - "from rag_loader import RagQueryLoader\n", + "from torch_geometric.loader import RAGQueryLoader\n", "from torch_geometric.datasets import UpdatedWebQSPDataset\n", "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet\n", + "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", "from torch_geometric.data import get_features_for_triplets_groups, Data\n", "from itertools import chain\n", - "import torch" + "import torch\n", + "from typing import Tuple" ] }, { @@ -52,8 +53,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "10it [00:00, 59.04it/s]00:00 Tuple[Data, str]:\n", + " q_emb = model.encode(query)\n", + " textual_nodes = ds.textual_nodes.iloc[graph[\"node_idx\"]].reset_index()\n", + " textual_edges = ds.textual_edges.iloc[graph[\"edge_idx\"]].reset_index()\n", + " out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e)\n", + " out_graph[\"desc\"] = desc\n", + " return graph" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={\"k_nodes\": 10}, seed_edges_kwargs={\"k_edges\": 10}, sampler_kwargs={\"num_neighbors\": [40]*10}, local_filter=apply_retrieval_via_pcst)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.5534146021759078 0.0007042666823170373 0.6\n", + "0.5555118626294403 0.00017691808692575338 0.75\n", + "0.5525756979944947 0.0001757881167233095 0.42857142857142855\n", + "0.5580547909293485 0.00011862396204033214 1.0\n", + "0.5605977192292568 5.966587112171838e-05 0.3333333333333333\n", + "0.5426923581072225 0.00017200848575196377 0.3333333333333333\n", + "0.5530475815965396 0.00011729517330361856 1.0\n", + "0.553126228863547 0.0002933239469670304 0.5\n", + "0.5531000131078778 0.00029337557941676935 0.35714285714285715\n", + "0.5271464149954123 5.545389009038984e-05 0.16666666666666666\n" + ] + } + ], + "source": [ + "for subg, gt in zip((query_loader.query(q) for q in questions), ds):\n", + " print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt))" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 3946e0d2c3d5..4bdd5ecfa718 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -150,6 +150,13 @@ def retrieval_via_pcst( data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes)) + # HACK Added so that the subset of nodes and edges selected can be tracked + try: + data['node_idx'] = np.array(graph.node_idx)[selected_nodes] + data['edge_idx'] = np.array(graph.edge_idx)[selected_edges] + except Exception: + pass + return data, desc diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 6615568d2151..f4e87f83404b 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -53,7 +53,9 @@ def register_feature_store(self, feature_store: FeatureStore): class RAGQueryLoader: def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], - local_filter: Optional[Callable] = None, + local_filter: Optional[Callable[[Data, Any], Data]] = None, + seed_nodes_kwargs: Optional[Dict[str, Any]] = None, + seed_edges_kwargs: Optional[Dict[str, Any]] = None, sampler_kwargs: Optional[Dict[str, Any]] = None, loader_kwargs: Optional[Dict[str, Any]] = None): fstore, gstore = data @@ -61,6 +63,8 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], self.graph_store = gstore self.graph_store.register_feature_store(self.feature_store) self.local_filter = local_filter + self.seed_nodes_kwargs = seed_nodes_kwargs or {} + self.seed_edges_kwargs = seed_edges_kwargs or {} self.sampler_kwargs = sampler_kwargs or {} self.loader_kwargs = loader_kwargs or {} @@ -68,8 +72,10 @@ def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature attributes. """ - seed_nodes = self.feature_store.retrieve_seed_nodes(query) - seed_edges = self.feature_store.retrieve_seed_edges(query) + seed_nodes = self.feature_store.retrieve_seed_nodes( + query, **self.seed_nodes_kwargs) + seed_edges = self.feature_store.retrieve_seed_edges( + query, **self.seed_edges_kwargs) subgraph_sample = self.graph_store.sample_subgraph( seed_nodes, seed_edges, **self.sampler_kwargs) @@ -78,5 +84,5 @@ def query(self, query: Any) -> Data: **self.loader_kwargs) if self.local_filter: - data = self.local_filter(data) + data = self.local_filter(data, query) return data From 53ab64a74007cf1353197ae294cc2afb10ce1f98 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 17 Jun 2024 18:30:17 -0700 Subject: [PATCH 624/752] fix saving bug for pcst hack --- torch_geometric/datasets/updated_web_qsp_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 4bdd5ecfa718..c489151407a4 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -47,6 +47,7 @@ def retrieval_via_pcst( topk: int = 3, topk_e: int = 3, cost_e: float = 0.5, + save_idx: bool = False, ) -> Tuple[Data, str]: c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: @@ -151,11 +152,9 @@ def retrieval_via_pcst( num_nodes=len(selected_nodes)) # HACK Added so that the subset of nodes and edges selected can be tracked - try: + if save_idx: data['node_idx'] = np.array(graph.node_idx)[selected_nodes] data['edge_idx'] = np.array(graph.edge_idx)[selected_edges] - except Exception: - pass return data, desc @@ -178,6 +177,7 @@ def __init__( self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) + self._load_raw_data() self.load(self.processed_paths[0]) def _check_dependencies(self) -> None: From c1fae6f74fbea96d49c365f29268792f0b1002b0 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 17 Jun 2024 18:50:48 -0700 Subject: [PATCH 625/752] fix bug in profiler and make blank file to be filled with unittests --- torch_geometric/profile/profiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/torch_geometric/profile/profiler.py b/torch_geometric/profile/profiler.py index 0ea8f73ad584..4b1376f54998 100644 --- a/torch_geometric/profile/profiler.py +++ b/torch_geometric/profile/profiler.py @@ -1,6 +1,7 @@ import functools from collections import OrderedDict, defaultdict, namedtuple from typing import Any, List, NamedTuple, Optional, Tuple +import re import torch import torch.profiler as torch_profiler @@ -302,13 +303,14 @@ def _layer_trace( layer_names = [] layer_stats = [] for format_line in format_lines: # body + # get current line's level based on number of '-' in prefix + current_line_level = len(re.match(r'^(-+).*', format_line[0]).group(1)) if format_line[1] == '': # key line - key_dict[format_line[0].count("-")] = format_line[0] + key_dict[current_line_level] = format_line[0] else: # must print # get current line's level - curr_level = format_line[0].count("-") par_str = "" - for i in range(1, curr_level): + for i in range(1, current_line_level): par_str += key_dict[i] curr_key = par_str + format_line[0] layer_names.append(curr_key) From a051259cbcc07d94192f2a83b202b9714e4aea67 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 17 Jun 2024 20:15:57 -0700 Subject: [PATCH 626/752] overhaul test profiling --- examples/llm_plus_gnn/test_profiling.ipynb | 26594 ++++++++++++++++++- 1 file changed, 26311 insertions(+), 283 deletions(-) diff --git a/examples/llm_plus_gnn/test_profiling.ipynb b/examples/llm_plus_gnn/test_profiling.ipynb index 153e628407e9..c879c654ef25 100644 --- a/examples/llm_plus_gnn/test_profiling.ipynb +++ b/examples/llm_plus_gnn/test_profiling.ipynb @@ -2,443 +2,26471 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ + "from torch_geometric.nn import GRetriever\n", "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from raw_qsp_dataset import RawWebQSPDataset\n", - "from torch_geometric.profile import profileit, timeit\n", - "from torch_geometric.profile.profile import GPUStats\n", - "from torch.profiler import profile\n", - "from typing import Protocol, Type, List, Tuple, Any\n", - "from abc import abstractmethod\n", - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class Profilable(Protocol):\n", - " model: torch.nn.Module\n", - " device: torch.device\n", - "\n", - " @abstractmethod\n", - " def _build_graph(self) -> None:\n", - " pass\n", - "\n", - " @abstractmethod\n", - " def _retrieve_subgraphs(self) -> None:\n", - " pass\n", - "\n", - "def make_profilable(dataset_obj: Type[Profilable]) -> Type[Profilable]:\n", - " dec = profileit(\"cuda\")\n", - "\n", - " class ProfilableObject(dataset_obj):\n", - " def __init__(self, *args, **kwargs) -> None:\n", - " self.desc = dict()\n", - " self.parent_cls = super()\n", - " self.parent_cls.__init__(*args, **kwargs)\n", - "\n", - " def _build_graph(self) -> None:\n", - " device_tensor = torch.Tensor().to(self.device)\n", - " wrap = dec(lambda model, dev_tensor: self.parent_cls._build_graph())\n", - " ret, desc = wrap(self.model, device_tensor)\n", - " self.desc['_build_graph'] = desc\n", - " return ret\n", - " \n", - " def _retrieve_subgraphs(self) -> None:\n", - " device_tensor = torch.Tensor().to(self.device)\n", - " wrap = dec(lambda model, dev_tensor: self.parent_cls._retrieve_subgraphs())\n", - " ret, desc = wrap(self.model, device_tensor)\n", - " self.desc['_retrieve_subgraphs'] = desc\n", - " return ret\n", - " \n", - " return ProfilableObject" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds = UpdatedWebQSPDataset(root=\"profiled_ds\", force_reload=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds = UpdatedWebQSPDataset(root=\"profiled_ds_wholegraph\", force_reload=True, whole_graph_retrieval=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "profilable_ds: Type[UpdatedWebQSPDataset] = make_profilable(UpdatedWebQSPDataset)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset: UpdatedWebQSPDataset = profilable_ds(root=\"profiled_ds\", force_reload=True, limit=100)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset.desc['_retrieve_subgraphs']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with profile(profile_memory=True, with_stack=True, record_shapes=True) as prof:\n", - " ds = UpdatedWebQSPDataset(root=\"profiled_ds\", force_reload=True, limit=10)\n", - " del ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prof.export_chrome_trace('timeline.json')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prof.export_memory_timeline('timeline_mem.html')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "avgs = prof.key_averages()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(avgs.table(sort_by='cpu_time'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from torchmetrics.functional import pairwise_cosine_similarity\n", + "from torch_geometric.profile.profiler import Profiler\n", + "from g_retriever import train, get_loss, inference_step\n", + "import time\n", "import torch" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "i1 = torch.rand((4700,1024)).to(device)\n", - "i2 = torch.rand((1000000,1024)).to(device)\n", - "i1 = pairwise_cosine_similarity(i1, i2)" + "ds = UpdatedWebQSPDataset(root=\"profiled_ds\")" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" + "Loading TinyLlama/TinyLlama-1.1B-Chat-v0.1\n", + "Setting up TinyLlama/TinyLlama-1.1B-Chat-v0.1 w/ kwargs = {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n" ] } ], "source": [ - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from torch_geometric.data import LargeGraphIndexer\n", - "from itertools import chain" + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = GRetriever(\n", + " llm_to_use=\"TinyLlama/TinyLlama-1.1B-Chat-v0.1\",\n", + " num_llm_params=1, # 1 Billion\n", + " gnn_hidden_channels=1024,\n", + " num_gnn_layers=4,\n", + " mlp_out_dim=2048,\n", + ").to(device)" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Prep Time (prep_time) = 0.3\n", + "Training beginning...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch: 1|1: 0%| | 0/353 [00:00 HeteroSamplerOutput | SamplerOutput:\n", - " if index.input_type is not None: # Heterogeneous\n", - " out = HeteroSamplerOutput(node={index.input_type: index.node}, row=dict(), col=dict(), edge=dict())\n", - " else:\n", - " out = SamplerOutput(node=index.node, row=torch.Tensor(), col=torch.Tensor())\n", - " return out\n", - " \n", - " def sample_from_edges(self, index: EdgeSamplerInput, neg_sampling: NegativeSampling | None = None) -> HeteroSamplerOutput | SamplerOutput:\n", - " EdgeSamplerInput()\n", - " if index.input_type is not None: # Heterogeneous\n", - " out = HeteroSamplerOutput(node=index.)" - ] - }, - { - "cell_type": "code", - "execution_count": 141, - "metadata": {}, - "outputs": [], - "source": [ - "link_sampler = NeighborSampler(data=graph, num_neighbors=[1], replace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 142, - "metadata": {}, - "outputs": [], - "source": [ - "load = LinkLoader(graph, link_sampler=link_sampler)" - ] - }, - { - "cell_type": "code", - "execution_count": 159, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "11466" + "('Module | Self CPU total | CPU total | Number of Calls\\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|-----------|----------------\\n-GAT | | | \\n-dropout | | | \\n--aten::dropout | 136.000us | 136.000us | 1413 \\n--cudaDeviceSynchronize | 4.750ms | 4.750ms | 1413 \\n-act | | | \\n--aten::relu | 280.710ms | 520.573ms | 1413 \\n--aten::clamp_min | 22.064ms | 239.863ms | 1413 \\n--cudaLaunchKernel | 217.799ms | 217.799ms | 1413 \\n--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 1413 \\n--cudaDeviceSynchronize | 3.419ms | 3.419ms | 1413 \\n-convs | | | \\n--0 | | | \\n---aggr_module | | | \\n----aten::view | 1.424ms | 1.424ms | 471 \\n----aten::expand_as | 615.000us | 1.819ms | 471 \\n----aten::expand | 980.000us | 1.365ms | 471 \\n----aten::as_strided | 440.000us | 440.000us | 1413 \\n----aten::new_zeros | 3.391ms | 199.119ms | 471 \\n----aten::new_empty | 4.615ms | 102.671ms | 471 \\n----aten::empty | 101.948ms | 101.948ms | 471 \\n----aten::zero_ | 1.214ms | 95.019ms | 471 \\n----aten::fill_ | 3.142ms | 93.805ms | 471 \\n----cudaLaunchKernel | 93.714ms | 93.714ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.842ms | 10.946ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.265ms | 1.265ms | 471 \\n---lin | | | \\n----aten::linear | 1.920ms | 230.667ms | 471 \\n----aten::t | 2.160ms | 3.650ms | 471 \\n----aten::transpose | 1.015ms | 1.490ms | 471 \\n----aten::as_strided | 475.000us | 475.000us | 471 \\n----aten::matmul | 1.212ms | 225.097ms | 471 \\n----aten::mm | 115.602ms | 223.885ms | 471 \\n----cudaFree | 5.856ms | 5.856ms | 2 \\n----cudaDeviceGetAttribute | 0.000us | 0.000us | 14 \\n----cudaGetSymbolAddress | 76.000us | 76.000us | 1 \\n----cudaMalloc | 183.000us | 183.000us | 3 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 984.000us | 984.000us | 241 \\n----cudaLaunchKernel | 92.051ms | 92.051ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.606ms | 1.606ms | 471 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 9.133ms | 9.133ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--1 | | | \\n---aggr_module | | | \\n----aten::view | 1.417ms | 1.417ms | 471 \\n----aten::expand_as | 556.000us | 1.754ms | 471 \\n----aten::expand | 976.000us | 1.364ms | 471 \\n----aten::as_strided | 444.000us | 444.000us | 1413 \\n----aten::new_zeros | 2.643ms | 181.493ms | 471 \\n----aten::new_empty | 2.877ms | 101.409ms | 471 \\n----aten::empty | 100.778ms | 100.778ms | 471 \\n----aten::zero_ | 1.140ms | 78.757ms | 471 \\n----aten::fill_ | 3.102ms | 77.617ms | 471 \\n----cudaLaunchKernel | 77.346ms | 77.346ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.741ms | 10.628ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.128ms | 1.128ms | 471 \\n---lin | | | \\n----aten::linear | 9.496ms | 194.940ms | 471 \\n----aten::t | 2.135ms | 3.325ms | 471 \\n----aten::transpose | 709.000us | 1.190ms | 471 \\n----aten::as_strided | 481.000us | 481.000us | 471 \\n----aten::matmul | 1.116ms | 190.137ms | 471 \\n----aten::mm | 104.854ms | 189.021ms | 471 \\n----cudaLaunchKernel | 75.774ms | 75.774ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.318ms | 1.318ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 208.000us | 208.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 8.185ms | 8.185ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--2 | | | \\n---aggr_module | | | \\n----aten::view | 1.422ms | 1.422ms | 471 \\n----aten::expand_as | 516.000us | 1.799ms | 471 \\n----aten::expand | 982.000us | 1.374ms | 471 \\n----aten::as_strided | 447.000us | 447.000us | 1413 \\n----aten::new_zeros | 4.523ms | 179.475ms | 471 \\n----aten::new_empty | 1.322ms | 110.879ms | 471 \\n----aten::empty | 110.271ms | 110.271ms | 471 \\n----aten::zero_ | 1.103ms | 67.282ms | 471 \\n----aten::fill_ | 3.111ms | 66.179ms | 471 \\n----cudaLaunchKernel | 65.813ms | 65.813ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.691ms | 10.490ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.046ms | 1.046ms | 471 \\n---lin | | | \\n----aten::linear | 7.860ms | 176.679ms | 471 \\n----aten::t | 2.104ms | 3.281ms | 471 \\n----aten::transpose | 700.000us | 1.177ms | 471 \\n----aten::as_strided | 477.000us | 477.000us | 471 \\n----aten::matmul | 1.124ms | 171.989ms | 471 \\n----aten::mm | 99.906ms | 170.865ms | 471 \\n----cudaLaunchKernel | 64.022ms | 64.022ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.434ms | 1.434ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 178.000us | 178.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 6.759ms | 6.759ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--3 | | | \\n---aggr_module | | | \\n----aten::view | 1.421ms | 1.421ms | 471 \\n----aten::expand_as | 539.000us | 1.889ms | 471 \\n----aten::expand | 1.020ms | 1.431ms | 471 \\n----aten::as_strided | 480.000us | 480.000us | 1413 \\n----aten::new_zeros | 3.339ms | 156.911ms | 471 \\n----aten::new_empty | 1.282ms | 93.703ms | 471 \\n----aten::empty | 93.156ms | 93.156ms | 471 \\n----aten::zero_ | 1.116ms | 61.915ms | 471 \\n----aten::fill_ | 3.116ms | 60.799ms | 471 \\n----cudaLaunchKernel | 60.432ms | 60.432ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.711ms | 10.529ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.044ms | 1.044ms | 471 \\n---lin | | | \\n----aten::linear | 10.282ms | 164.389ms | 471 \\n----aten::t | 2.049ms | 3.278ms | 471 \\n----aten::transpose | 753.000us | 1.229ms | 471 \\n----aten::as_strided | 476.000us | 476.000us | 471 \\n----aten::matmul | 1.070ms | 159.753ms | 471 \\n----aten::mm | 95.770ms | 158.683ms | 471 \\n----cudaStreamIsCapturing | 1.000us | 1.000us | 1 \\n----cudaMalloc | 152.000us | 152.000us | 1 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 1.023ms | 1.023ms | 511 \\n----cudaLaunchKernel | 32.548ms | 32.548ms | 471 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 172 \\n----cudaDeviceSynchronize | 6.656ms | 6.656ms | 471 \\n----cudaMemsetAsync | 29.189ms | 29.189ms | 230 \\n----Memset (Device) | 0.000us | 0.000us | 230 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 214 \\n----ampere_sgemm_128x32_sliced1x4_tn | 0.000us | 0.000us | 83 \\n----ampere_sgemm_128x64_tn | 0.000us | 0.000us | 2 \\n-norms | | | \\n--0 | | | \\n---module | | | \\n----aten::add_ | 79.108ms | 143.631ms | 353 \\n----cudaLaunchKernel | 74.679ms | 74.679ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 1.191ms | 84.025ms | 471 \\n----aten::_batch_norm_impl_index | 2.948ms | 82.488ms | 471 \\n----aten::empty | 30.383ms | 30.383ms | 1413 \\n----aten::native_batch_norm | 12.604ms | 52.167ms | 471 \\n----aten::empty_like | 750.000us | 3.136ms | 471 \\n----aten::empty_strided | 2.504ms | 2.504ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.285ms | 1.285ms | 471 \\n----aten::copy_ | 747.000us | 23.627ms | 118 \\n----cudaMemcpyAsync | 22.880ms | 22.880ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--1 | | | \\n---module | | | \\n----aten::add_ | 87.275ms | 139.715ms | 353 \\n----cudaLaunchKernel | 59.714ms | 59.714ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 528.000us | 77.774ms | 471 \\n----aten::_batch_norm_impl_index | 3.016ms | 76.477ms | 471 \\n----aten::empty | 32.189ms | 32.189ms | 1413 \\n----aten::native_batch_norm | 11.195ms | 44.783ms | 471 \\n----aten::empty_like | 634.000us | 3.093ms | 471 \\n----aten::empty_strided | 2.520ms | 2.520ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.066ms | 1.066ms | 471 \\n----aten::copy_ | 745.000us | 20.607ms | 118 \\n----cudaMemcpyAsync | 19.862ms | 19.862ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--2 | | | \\n---module | | | \\n-- -aten::add_ | 86.501ms | 132.128ms | 353 \\n-- -cudaLaunchKernel | 52.722ms | 52.722ms | 1648 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n-- -aten::batch_norm | 873.000us | 72.646ms | 471 \\n-- -aten::_batch_norm_impl_index | 2.720ms | 71.364ms | 471 \\n-- -aten::empty | 30.893ms | 30.893ms | 1413 \\n-- -aten::native_batch_norm | 10.998ms | 40.990ms | 471 \\n-- -aten::empty_like | 632.000us | 3.084ms | 471 \\n-- -aten::empty_strided | 2.510ms | 2.510ms | 471 \\n-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n-- -cudaDeviceSynchronize | 1.048ms | 1.048ms | 471 \\n-- -aten::copy_ | 745.000us | 17.173ms | 118 \\n-- -cudaMemcpyAsync | 16.428ms | 16.428ms | 118 \\n-- -Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--3 | | | \\n-_trim | | | \\n',\n", + " ['Module',\n", + " 'Self CPU total',\n", + " 'CPU total',\n", + " 'Self CUDA total',\n", + " 'CUDA total',\n", + " 'Self CPU Mem',\n", + " 'CPU Mem',\n", + " 'Self CUDA Mem',\n", + " 'CUDA Mem',\n", + " 'Number of Calls'],\n", + " {'aten::dropout': [136, 136, 0, 0, 0, 0, 0, 0, 1413],\n", + " 'cudaDeviceSynchronize': [1048, 1048, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::relu': [280710, 520573, 0, 1413, 0, 0, 0, 0, 1413],\n", + " 'aten::clamp_min': [22064, 239863, 1413, 1413, 0, 0, 0, 0, 1413],\n", + " 'cudaLaunchKernel': [52722, 52722, 0, 0, 0, 0, 0, 0, 1648],\n", + " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", + " 0,\n", + " 1413,\n", + " 1413,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 1413],\n", + " 'aten::view': [1421, 1421, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::expand_as': [539, 1889, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::expand': [1020, 1431, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::as_strided': [476, 476, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::new_zeros': [3339, 156911, 0, 367, 0, 0, 0, 0, 471],\n", + " 'aten::new_empty': [1282, 93703, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::empty': [30893, 30893, 0, 0, 0, 0, 0, 0, 1413],\n", + " 'aten::zero_': [1116, 61915, 0, 381, 0, 0, 0, 0, 471],\n", + " 'aten::fill_': [3116, 60799, 381, 381, 0, 0, 0, 0, 471],\n", + " 'void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)': [0,\n", + " 0,\n", + " 381,\n", + " 381,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 471],\n", + " 'aten::scatter_add_': [7711, 10529, 1307, 1307, 0, 0, 0, 0, 471],\n", + " 'void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})': [0,\n", + " 0,\n", + " 1307,\n", + " 1307,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 471],\n", + " 'aten::linear': [10282, 164389, 0, 16237, 0, 0, 0, 0, 471],\n", + " 'aten::t': [2049, 3278, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::transpose': [753, 1229, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::matmul': [1070, 159753, 0, 17112, 0, 0, 0, 0, 471],\n", + " 'aten::mm': [95770, 158683, 17112, 17112, 0, 0, 0, 0, 471],\n", + " 'cudaFree': [5856, 5856, 0, 0, 0, 0, 0, 0, 2],\n", + " 'cudaDeviceGetAttribute': [0, 0, 0, 0, 0, 0, 0, 0, 14],\n", + " 'cudaGetSymbolAddress': [76, 76, 0, 0, 0, 0, 0, 0, 1],\n", + " 'cudaMalloc': [152, 152, 0, 0, 0, 0, 0, 0, 1],\n", + " 'cudaOccupancyMaxActiveBlocksPerMultiprocessor': [1023,\n", + " 1023,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 511],\n", + " 'ampere_sgemm_128x32_tn': [0, 0, 9892, 9892, 0, 0, 0, 0, 214],\n", + " 'void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)': [0,\n", + " 0,\n", + " 456,\n", + " 456,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 425],\n", + " 'ampere_sgemm_64x32_sliced1x4_tn': [0, 0, 1482, 1482, 0, 0, 0, 0, 132],\n", + " 'ampere_sgemm_32x32_sliced1x4_tn': [0, 0, 4233, 4233, 0, 0, 0, 0, 172],\n", + " 'cudaMemsetAsync': [29189, 29189, 0, 0, 0, 0, 0, 0, 230],\n", + " 'Memset (Device)': [0, 0, 0, 0, 0, 0, 0, 0, 230],\n", + " 'cudaStreamIsCapturing': [1, 1, 0, 0, 0, 0, 0, 0, 1],\n", + " 'ampere_sgemm_128x32_sliced1x4_tn': [0, 0, 2876, 2876, 0, 0, 0, 0, 83],\n", + " 'ampere_sgemm_128x64_tn': [0, 0, 111, 111, 0, 0, 0, 0, 2],\n", + " 'aten::add_': [86501, 132128, 353, 353, 0, 0, 0, 0, 353],\n", + " 'void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)': [0,\n", + " 0,\n", + " 353,\n", + " 353,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 353],\n", + " 'aten::batch_norm': [873, 72646, 0, 2573, 0, 0, 0, 0, 471],\n", + " 'aten::_batch_norm_impl_index': [2720, 71364, 0, 2539, 0, 0, 0, 0, 471],\n", + " 'aten::native_batch_norm': [10998, 40990, 2479, 2597, 0, 0, 0, 0, 471],\n", + " 'aten::empty_like': [632, 3084, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::empty_strided': [2510, 2510, 0, 0, 0, 0, 0, 0, 471],\n", + " 'void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)': [0,\n", + " 0,\n", + " 1066,\n", + " 1066,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 353],\n", + " 'void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)': [0,\n", + " 0,\n", + " 353,\n", + " 353,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 353],\n", + " 'void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)': [0,\n", + " 0,\n", + " 942,\n", + " 942,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 471],\n", + " 'aten::copy_': [745, 17173, 118, 118, 0, 0, 0, 0, 118],\n", + " 'cudaMemcpyAsync': [16428, 16428, 0, 0, 0, 0, 0, 0, 118],\n", + " 'Memcpy DtoD (Device -> Device)': [0, 0, 118, 118, 0, 0, 0, 0, 118],\n", + " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", + " 0,\n", + " 118,\n", + " 118,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 118]},\n", + " ['-dropout--aten::dropout',\n", + " '-dropout--cudaDeviceSynchronize',\n", + " '-act--aten::relu',\n", + " '-act--aten::clamp_min',\n", + " '-act--cudaLaunchKernel',\n", + " '-act--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", + " '-act--cudaDeviceSynchronize',\n", + " '-convs--0---aggr_module----aten::view',\n", + " '-convs--0---aggr_module----aten::expand_as',\n", + " '-convs--0---aggr_module----aten::expand',\n", + " '-convs--0---aggr_module----aten::as_strided',\n", + " '-convs--0---aggr_module----aten::new_zeros',\n", + " '-convs--0---aggr_module----aten::new_empty',\n", + " '-convs--0---aggr_module----aten::empty',\n", + " '-convs--0---aggr_module----aten::zero_',\n", + " '-convs--0---aggr_module----aten::fill_',\n", + " '-convs--0---aggr_module----cudaLaunchKernel',\n", + " '-convs--0---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", + " '-convs--0---aggr_module----aten::scatter_add_',\n", + " '-convs--0---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", + " '-convs--0---aggr_module----cudaDeviceSynchronize',\n", + " '-convs--0---lin----aten::linear',\n", + " '-convs--0---lin----aten::t',\n", + " '-convs--0---lin----aten::transpose',\n", + " '-convs--0---lin----aten::as_strided',\n", + " '-convs--0---lin----aten::matmul',\n", + " '-convs--0---lin----aten::mm',\n", + " '-convs--0---lin----cudaFree',\n", + " '-convs--0---lin----cudaDeviceGetAttribute',\n", + " '-convs--0---lin----cudaGetSymbolAddress',\n", + " '-convs--0---lin----cudaMalloc',\n", + " '-convs--0---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", + " '-convs--0---lin----cudaLaunchKernel',\n", + " '-convs--0---lin----ampere_sgemm_128x32_tn',\n", + " '-convs--0---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", + " '-convs--0---lin----cudaDeviceSynchronize',\n", + " '-convs--0---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", + " '-convs--0---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", + " '-convs--0---lin----cudaMemsetAsync',\n", + " '-convs--0---lin----Memset (Device)',\n", + " '-convs--1---aggr_module----aten::view',\n", + " '-convs--1---aggr_module----aten::expand_as',\n", + " '-convs--1---aggr_module----aten::expand',\n", + " '-convs--1---aggr_module----aten::as_strided',\n", + " '-convs--1---aggr_module----aten::new_zeros',\n", + " '-convs--1---aggr_module----aten::new_empty',\n", + " '-convs--1---aggr_module----aten::empty',\n", + " '-convs--1---aggr_module----aten::zero_',\n", + " '-convs--1---aggr_module----aten::fill_',\n", + " '-convs--1---aggr_module----cudaLaunchKernel',\n", + " '-convs--1---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", + " '-convs--1---aggr_module----aten::scatter_add_',\n", + " '-convs--1---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", + " '-convs--1---aggr_module----cudaDeviceSynchronize',\n", + " '-convs--1---lin----aten::linear',\n", + " '-convs--1---lin----aten::t',\n", + " '-convs--1---lin----aten::transpose',\n", + " '-convs--1---lin----aten::as_strided',\n", + " '-convs--1---lin----aten::matmul',\n", + " '-convs--1---lin----aten::mm',\n", + " '-convs--1---lin----cudaLaunchKernel',\n", + " '-convs--1---lin----ampere_sgemm_128x32_tn',\n", + " '-convs--1---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", + " '-convs--1---lin----cudaDeviceSynchronize',\n", + " '-convs--1---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", + " '-convs--1---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", + " '-convs--1---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", + " '-convs--1---lin----cudaMemsetAsync',\n", + " '-convs--1---lin----Memset (Device)',\n", + " '-convs--2---aggr_module----aten::view',\n", + " '-convs--2---aggr_module----aten::expand_as',\n", + " '-convs--2---aggr_module----aten::expand',\n", + " '-convs--2---aggr_module----aten::as_strided',\n", + " '-convs--2---aggr_module----aten::new_zeros',\n", + " '-convs--2---aggr_module----aten::new_empty',\n", + " '-convs--2---aggr_module----aten::empty',\n", + " '-convs--2---aggr_module----aten::zero_',\n", + " '-convs--2---aggr_module----aten::fill_',\n", + " '-convs--2---aggr_module----cudaLaunchKernel',\n", + " '-convs--2---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", + " '-convs--2---aggr_module----aten::scatter_add_',\n", + " '-convs--2---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", + " '-convs--2---aggr_module----cudaDeviceSynchronize',\n", + " '-convs--2---lin----aten::linear',\n", + " '-convs--2---lin----aten::t',\n", + " '-convs--2---lin----aten::transpose',\n", + " '-convs--2---lin----aten::as_strided',\n", + " '-convs--2---lin----aten::matmul',\n", + " '-convs--2---lin----aten::mm',\n", + " '-convs--2---lin----cudaLaunchKernel',\n", + " '-convs--2---lin----ampere_sgemm_128x32_tn',\n", + " '-convs--2---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", + " '-convs--2---lin----cudaDeviceSynchronize',\n", + " '-convs--2---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", + " '-convs--2---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", + " '-convs--2---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", + " '-convs--2---lin----cudaMemsetAsync',\n", + " '-convs--2---lin----Memset (Device)',\n", + " '-convs--3---aggr_module----aten::view',\n", + " '-convs--3---aggr_module----aten::expand_as',\n", + " '-convs--3---aggr_module----aten::expand',\n", + " '-convs--3---aggr_module----aten::as_strided',\n", + " '-convs--3---aggr_module----aten::new_zeros',\n", + " '-convs--3---aggr_module----aten::new_empty',\n", + " '-convs--3---aggr_module----aten::empty',\n", + " '-convs--3---aggr_module----aten::zero_',\n", + " '-convs--3---aggr_module----aten::fill_',\n", + " '-convs--3---aggr_module----cudaLaunchKernel',\n", + " '-convs--3---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", + " '-convs--3---aggr_module----aten::scatter_add_',\n", + " '-convs--3---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", + " '-convs--3---aggr_module----cudaDeviceSynchronize',\n", + " '-convs--3---lin----aten::linear',\n", + " '-convs--3---lin----aten::t',\n", + " '-convs--3---lin----aten::transpose',\n", + " '-convs--3---lin----aten::as_strided',\n", + " '-convs--3---lin----aten::matmul',\n", + " '-convs--3---lin----aten::mm',\n", + " '-convs--3---lin----cudaStreamIsCapturing',\n", + " '-convs--3---lin----cudaMalloc',\n", + " '-convs--3---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", + " '-convs--3---lin----cudaLaunchKernel',\n", + " '-convs--3---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", + " '-convs--3---lin----cudaDeviceSynchronize',\n", + " '-convs--3---lin----cudaMemsetAsync',\n", + " '-convs--3---lin----Memset (Device)',\n", + " '-convs--3---lin----ampere_sgemm_128x32_tn',\n", + " '-convs--3---lin----ampere_sgemm_128x32_sliced1x4_tn',\n", + " '-convs--3---lin----ampere_sgemm_128x64_tn',\n", + " '-norms--0---module----aten::add_',\n", + " '-norms--0---module----cudaLaunchKernel',\n", + " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", + " '-norms--0---module----aten::batch_norm',\n", + " '-norms--0---module----aten::_batch_norm_impl_index',\n", + " '-norms--0---module----aten::empty',\n", + " '-norms--0---module----aten::native_batch_norm',\n", + " '-norms--0---module----aten::empty_like',\n", + " '-norms--0---module----aten::empty_strided',\n", + " '-norms--0---module----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", + " '-norms--0---module----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", + " '-norms--0---module----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", + " '-norms--0---module----cudaDeviceSynchronize',\n", + " '-norms--0---module----aten::copy_',\n", + " '-norms--0---module----cudaMemcpyAsync',\n", + " '-norms--0---module----Memcpy DtoD (Device -> Device)',\n", + " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", + " '-norms--1---module----aten::add_',\n", + " '-norms--1---module----cudaLaunchKernel',\n", + " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", + " '-norms--1---module----aten::batch_norm',\n", + " '-norms--1---module----aten::_batch_norm_impl_index',\n", + " '-norms--1---module----aten::empty',\n", + " '-norms--1---module----aten::native_batch_norm',\n", + " '-norms--1---module----aten::empty_like',\n", + " '-norms--1---module----aten::empty_strided',\n", + " '-norms--1---module----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", + " '-norms--1---module----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", + " '-norms--1---module----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", + " '-norms--1---module----cudaDeviceSynchronize',\n", + " '-norms--1---module----aten::copy_',\n", + " '-norms--1---module----cudaMemcpyAsync',\n", + " '-norms--1---module----Memcpy DtoD (Device -> Device)',\n", + " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", + " '-norms-- -aten::add_',\n", + " '-norms-- -cudaLaunchKernel',\n", + " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", + " '-norms-- -aten::batch_norm',\n", + " '-norms-- -aten::_batch_norm_impl_index',\n", + " '-norms-- -aten::empty',\n", + " '-norms-- -aten::native_batch_norm',\n", + " '-norms-- -aten::empty_like',\n", + " '-norms-- -aten::empty_strided',\n", + " '-norms-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", + " '-norms-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", + " '-norms-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", + " '-norms-- -cudaDeviceSynchronize',\n", + " '-norms-- -aten::copy_',\n", + " '-norms-- -cudaMemcpyAsync',\n", + " '-norms-- -Memcpy DtoD (Device -> Device)',\n", + " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)'],\n", + " [['136.000us',\n", + " '136.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['4.750ms',\n", + " '4.750ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['280.710ms',\n", + " '520.573ms',\n", + " '0.000us',\n", + " '1.413ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['22.064ms',\n", + " '239.863ms',\n", + " '1.413ms',\n", + " '1.413ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['217.799ms',\n", + " '217.799ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.413ms',\n", + " '1.413ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['3.419ms',\n", + " '3.419ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['1.424ms',\n", + " '1.424ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['615.000us',\n", + " '1.819ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['980.000us',\n", + " '1.365ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['440.000us',\n", + " '440.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['3.391ms',\n", + " '199.119ms',\n", + " '0.000us',\n", + " '35.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['4.615ms',\n", + " '102.671ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['101.948ms',\n", + " '101.948ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.214ms',\n", + " '95.019ms',\n", + " '0.000us',\n", + " '39.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['3.142ms',\n", + " '93.805ms',\n", + " '39.000us',\n", + " '39.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['93.714ms',\n", + " '93.714ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '942'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '39.000us',\n", + " '39.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['7.842ms',\n", + " '10.946ms',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.265ms',\n", + " '1.265ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.920ms',\n", + " '230.667ms',\n", + " '0.000us',\n", + " '6.433ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.160ms',\n", + " '3.650ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.015ms',\n", + " '1.490ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['475.000us',\n", + " '475.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.212ms',\n", + " '225.097ms',\n", + " '0.000us',\n", + " '6.433ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['115.602ms',\n", + " '223.885ms',\n", + " '6.433ms',\n", + " '6.433ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['5.856ms',\n", + " '5.856ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '2'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '14'],\n", + " ['76.000us',\n", + " '76.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1'],\n", + " ['183.000us',\n", + " '183.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '3'],\n", + " ['984.000us',\n", + " '984.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '241'],\n", + " ['92.051ms',\n", + " '92.051ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '896'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '839.000us',\n", + " '839.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '98'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '511.000us',\n", + " '511.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '425'],\n", + " ['1.606ms',\n", + " '1.606ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.767ms',\n", + " '1.767ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '132'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '3.316ms',\n", + " '3.316ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '241'],\n", + " ['9.133ms',\n", + " '9.133ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['1.417ms',\n", + " '1.417ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['556.000us',\n", + " '1.754ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['976.000us',\n", + " '1.364ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['444.000us',\n", + " '444.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['2.643ms',\n", + " '181.493ms',\n", + " '0.000us',\n", + " '3.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.877ms',\n", + " '101.409ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['100.778ms',\n", + " '100.778ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.140ms',\n", + " '78.757ms',\n", + " '0.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['3.102ms',\n", + " '77.617ms',\n", + " '4.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['77.346ms',\n", + " '77.346ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '942'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '4.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['7.741ms',\n", + " '10.628ms',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.128ms',\n", + " '1.128ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['9.496ms',\n", + " '194.940ms',\n", + " '0.000us',\n", + " '5.475ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.135ms',\n", + " '3.325ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['709.000us',\n", + " '1.190ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['481.000us',\n", + " '481.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.116ms',\n", + " '190.137ms',\n", + " '0.000us',\n", + " '5.671ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['104.854ms',\n", + " '189.021ms',\n", + " '5.671ms',\n", + " '5.671ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['75.774ms',\n", + " '75.774ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '896'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '692.000us',\n", + " '692.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '98'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '459.000us',\n", + " '459.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '425'],\n", + " ['1.318ms',\n", + " '1.318ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['208.000us',\n", + " '208.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '132'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.481ms',\n", + " '1.481ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '132'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '3.039ms',\n", + " '3.039ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '241'],\n", + " ['8.185ms',\n", + " '8.185ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['1.422ms',\n", + " '1.422ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['516.000us',\n", + " '1.799ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['982.000us',\n", + " '1.374ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['447.000us',\n", + " '447.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['4.523ms',\n", + " '179.475ms',\n", + " '0.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.322ms',\n", + " '110.879ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['110.271ms',\n", + " '110.271ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.103ms',\n", + " '67.282ms',\n", + " '0.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['3.111ms',\n", + " '66.179ms',\n", + " '4.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['65.813ms',\n", + " '65.813ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '942'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '4.000us',\n", + " '4.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['7.691ms',\n", + " '10.490ms',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.046ms',\n", + " '1.046ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['7.860ms',\n", + " '176.679ms',\n", + " '0.000us',\n", + " '5.494ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.104ms',\n", + " '3.281ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['700.000us',\n", + " '1.177ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['477.000us',\n", + " '477.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.124ms',\n", + " '171.989ms',\n", + " '0.000us',\n", + " '5.675ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['99.906ms',\n", + " '170.865ms',\n", + " '5.675ms',\n", + " '5.675ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['64.022ms',\n", + " '64.022ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '896'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '696.000us',\n", + " '696.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '98'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '456.000us',\n", + " '456.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '425'],\n", + " ['1.434ms',\n", + " '1.434ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['178.000us',\n", + " '178.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '132'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.482ms',\n", + " '1.482ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '132'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '3.041ms',\n", + " '3.041ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '241'],\n", + " ['6.759ms',\n", + " '6.759ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '44'],\n", + " ['1.421ms',\n", + " '1.421ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['539.000us',\n", + " '1.889ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.020ms',\n", + " '1.431ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['480.000us',\n", + " '480.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['3.339ms',\n", + " '156.911ms',\n", + " '0.000us',\n", + " '367.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.282ms',\n", + " '93.703ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['93.156ms',\n", + " '93.156ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.116ms',\n", + " '61.915ms',\n", + " '0.000us',\n", + " '381.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['3.116ms',\n", + " '60.799ms',\n", + " '381.000us',\n", + " '381.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['60.432ms',\n", + " '60.432ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '942'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '381.000us',\n", + " '381.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['7.711ms',\n", + " '10.529ms',\n", + " '1.307ms',\n", + " '1.307ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.307ms',\n", + " '1.307ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.044ms',\n", + " '1.044ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['10.282ms',\n", + " '164.389ms',\n", + " '0.000us',\n", + " '16.237ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.049ms',\n", + " '3.278ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['753.000us',\n", + " '1.229ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['476.000us',\n", + " '476.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.070ms',\n", + " '159.753ms',\n", + " '0.000us',\n", + " '17.112ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['95.770ms',\n", + " '158.683ms',\n", + " '17.112ms',\n", + " '17.112ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.000us',\n", + " '1.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1'],\n", + " ['152.000us',\n", + " '152.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1'],\n", + " ['1.023ms',\n", + " '1.023ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '511'],\n", + " ['32.548ms',\n", + " '32.548ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '4.233ms',\n", + " '4.233ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '172'],\n", + " ['6.656ms',\n", + " '6.656ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['29.189ms',\n", + " '29.189ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '230'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '230'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '9.892ms',\n", + " '9.892ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '214'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '2.876ms',\n", + " '2.876ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '83'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '111.000us',\n", + " '111.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '2'],\n", + " ['79.108ms',\n", + " '143.631ms',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['74.679ms',\n", + " '74.679ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1648'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['1.191ms',\n", + " '84.025ms',\n", + " '0.000us',\n", + " '2.582ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.948ms',\n", + " '82.488ms',\n", + " '0.000us',\n", + " '2.566ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['30.383ms',\n", + " '30.383ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['12.604ms',\n", + " '52.167ms',\n", + " '2.476ms',\n", + " '2.594ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['750.000us',\n", + " '3.136ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.504ms',\n", + " '2.504ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.064ms',\n", + " '1.064ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '941.000us',\n", + " '941.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.285ms',\n", + " '1.285ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['747.000us',\n", + " '23.627ms',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['22.880ms',\n", + " '22.880ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['87.275ms',\n", + " '139.715ms',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['59.714ms',\n", + " '59.714ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1648'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['528.000us',\n", + " '77.774ms',\n", + " '0.000us',\n", + " '2.578ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['3.016ms',\n", + " '76.477ms',\n", + " '0.000us',\n", + " '2.548ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['32.189ms',\n", + " '32.189ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['11.195ms',\n", + " '44.783ms',\n", + " '2.478ms',\n", + " '2.596ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['634.000us',\n", + " '3.093ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.520ms',\n", + " '2.520ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.065ms',\n", + " '1.065ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.066ms',\n", + " '1.066ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['745.000us',\n", + " '20.607ms',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['19.862ms',\n", + " '19.862ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['86.501ms',\n", + " '132.128ms',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['52.722ms',\n", + " '52.722ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1648'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['873.000us',\n", + " '72.646ms',\n", + " '0.000us',\n", + " '2.573ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.720ms',\n", + " '71.364ms',\n", + " '0.000us',\n", + " '2.539ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['30.893ms',\n", + " '30.893ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '1413'],\n", + " ['10.998ms',\n", + " '40.990ms',\n", + " '2.479ms',\n", + " '2.597ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['632.000us',\n", + " '3.084ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['2.510ms',\n", + " '2.510ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '1.066ms',\n", + " '1.066ms',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '353.000us',\n", + " '353.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '353'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '942.000us',\n", + " '942.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['1.048ms',\n", + " '1.048ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '471'],\n", + " ['745.000us',\n", + " '17.173ms',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['16.428ms',\n", + " '16.428ms',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118'],\n", + " ['0.000us',\n", + " '0.000us',\n", + " '118.000us',\n", + " '118.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '0 b',\n", + " '118']])" ] }, - "execution_count": 159, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "link_sampler.num_nodes" + "trace" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "load2 = LinkNeighborLoader(graph, num_neighbors=[2])" + "with open('trace.txt', 'w') as f:\n", + " f.write(trace[0])" ] }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "result = load([1])" + "import pandas as pd" ] }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([17505, 24971])" + "10" ] }, - "execution_count": 154, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "result.e_id" + "len(trace[1])" ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[10097, 4044, 5397, ..., 7188, 8251, 3597],\n", - " [10097, 673, 5827, ..., 7594, 673, 10951]])" + "['Module',\n", + " 'Self CPU total',\n", + " 'CPU total',\n", + " 'Self CUDA total',\n", + " 'CUDA total',\n", + " 'Self CPU Mem',\n", + " 'CPU Mem',\n", + " 'Self CUDA Mem',\n", + " 'CUDA Mem',\n", + " 'Number of Calls']" ] }, - "execution_count": 107, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "graph.edge_index" + "trace[1]" ] }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 21, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'m.09g3c50'" - ] - }, - "execution_count": 153, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ds.indexer.node_attr['pid'][4759]" + "df = pd.DataFrame(trace[4], columns=trace[1][1:], index=trace[3])" ] }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Self CPU totalCPU totalSelf CUDA totalCUDA totalSelf CPU MemCPU MemSelf CUDA MemCUDA MemNumber of Calls
-dropout--aten::dropout136.000us136.000us0.000us0.000us0 b0 b0 b0 b1413
-dropout--cudaDeviceSynchronize4.750ms4.750ms0.000us0.000us0 b0 b0 b0 b1413
-act--aten::relu280.710ms520.573ms0.000us1.413ms0 b0 b0 b0 b1413
-act--aten::clamp_min22.064ms239.863ms1.413ms1.413ms0 b0 b0 b0 b1413
-act--cudaLaunchKernel217.799ms217.799ms0.000us0.000us0 b0 b0 b0 b1413
..............................
-norms-- -cudaDeviceSynchronize1.048ms1.048ms0.000us0.000us0 b0 b0 b0 b471
-norms-- -aten::copy_745.000us17.173ms118.000us118.000us0 b0 b0 b0 b118
-norms-- -cudaMemcpyAsync16.428ms16.428ms0.000us0.000us0 b0 b0 b0 b118
-norms-- -Memcpy DtoD (Device -> Device)0.000us0.000us118.000us118.000us0 b0 b0 b0 b118
-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array<char*, 2> >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array<char*, 2>)0.000us0.000us118.000us118.000us0 b0 b0 b0 b118
\n", + "

180 rows × 9 columns

\n", + "
" + ], "text/plain": [ - "('washington redskins at oakland raiders, 2009-12-13',\n", - " 'american_football.football_game.away_team',\n", - " 'washington redskins')" + " Self CPU total CPU total \\\n", + "-dropout--aten::dropout 136.000us 136.000us \n", + "-dropout--cudaDeviceSynchronize 4.750ms 4.750ms \n", + "-act--aten::relu 280.710ms 520.573ms \n", + "-act--aten::clamp_min 22.064ms 239.863ms \n", + "-act--cudaLaunchKernel 217.799ms 217.799ms \n", + "... ... ... \n", + "-norms-- -cudaDeviceSynchronize 1.048ms 1.048ms \n", + "-norms-- -aten::copy_ 745.000us 17.173ms \n", + "-norms-- -cudaMemcpyAsync 16.428ms 16.428ms \n", + "-norms-- -Memcpy DtoD (Device -> Device) 0.000us 0.000us \n", + "-norms-- -void at::native::vectorized_elementwi... 0.000us 0.000us \n", + "\n", + " Self CUDA total CUDA total \\\n", + "-dropout--aten::dropout 0.000us 0.000us \n", + "-dropout--cudaDeviceSynchronize 0.000us 0.000us \n", + "-act--aten::relu 0.000us 1.413ms \n", + "-act--aten::clamp_min 1.413ms 1.413ms \n", + "-act--cudaLaunchKernel 0.000us 0.000us \n", + "... ... ... \n", + "-norms-- -cudaDeviceSynchronize 0.000us 0.000us \n", + "-norms-- -aten::copy_ 118.000us 118.000us \n", + "-norms-- -cudaMemcpyAsync 0.000us 0.000us \n", + "-norms-- -Memcpy DtoD (Device -> Device) 118.000us 118.000us \n", + "-norms-- -void at::native::vectorized_elementwi... 118.000us 118.000us \n", + "\n", + " Self CPU Mem CPU Mem \\\n", + "-dropout--aten::dropout 0 b 0 b \n", + "-dropout--cudaDeviceSynchronize 0 b 0 b \n", + "-act--aten::relu 0 b 0 b \n", + "-act--aten::clamp_min 0 b 0 b \n", + "-act--cudaLaunchKernel 0 b 0 b \n", + "... ... ... \n", + "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", + "-norms-- -aten::copy_ 0 b 0 b \n", + "-norms-- -cudaMemcpyAsync 0 b 0 b \n", + "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", + "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", + "\n", + " Self CUDA Mem CUDA Mem \\\n", + "-dropout--aten::dropout 0 b 0 b \n", + "-dropout--cudaDeviceSynchronize 0 b 0 b \n", + "-act--aten::relu 0 b 0 b \n", + "-act--aten::clamp_min 0 b 0 b \n", + "-act--cudaLaunchKernel 0 b 0 b \n", + "... ... ... \n", + "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", + "-norms-- -aten::copy_ 0 b 0 b \n", + "-norms-- -cudaMemcpyAsync 0 b 0 b \n", + "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", + "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", + "\n", + " Number of Calls \n", + "-dropout--aten::dropout 1413 \n", + "-dropout--cudaDeviceSynchronize 1413 \n", + "-act--aten::relu 1413 \n", + "-act--aten::clamp_min 1413 \n", + "-act--cudaLaunchKernel 1413 \n", + "... ... \n", + "-norms-- -cudaDeviceSynchronize 471 \n", + "-norms-- -aten::copy_ 118 \n", + "-norms-- -cudaMemcpyAsync 118 \n", + "-norms-- -Memcpy DtoD (Device -> Device) 118 \n", + "-norms-- -void at::native::vectorized_elementwi... 118 \n", + "\n", + "[180 rows x 9 columns]" ] }, - "execution_count": 155, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds.indexer.edge_attr[\"e_pid\"][1]" + "df" ] }, { From 3c60f03845b6416181fd5ffe38a7862b93f73009 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 25 Jun 2024 17:35:47 -0700 Subject: [PATCH 627/752] begin nvtx profiler --- examples/llm_plus_gnn/test_profiling.ipynb | 25344 +------------------ torch_geometric/profile/__init__.py | 2 + torch_geometric/profile/nvtx.py | 27 + 3 files changed, 804 insertions(+), 24569 deletions(-) create mode 100644 torch_geometric/profile/nvtx.py diff --git a/examples/llm_plus_gnn/test_profiling.ipynb b/examples/llm_plus_gnn/test_profiling.ipynb index c879c654ef25..8bba158b70d2 100644 --- a/examples/llm_plus_gnn/test_profiling.ipynb +++ b/examples/llm_plus_gnn/test_profiling.ipynb @@ -18,6 +18,7 @@ "from torch_geometric.nn import GRetriever\n", "from torch_geometric.datasets import UpdatedWebQSPDataset\n", "from torch_geometric.profile.profiler import Profiler\n", + "from torch_geometric.profile import profileit, timeit, nvtxit\n", "from g_retriever import train, get_loss, inference_step\n", "import time\n", "import torch" @@ -61,12 +62,58 @@ "cell_type": "code", "execution_count": 4, "metadata": {}, + "outputs": [], + "source": [ + "@nvtxit(\"test_inference_wrapper\")\n", + "def inference_wrapper(*args, **kwargs):\n", + " return inference_step(*args, **kwargs)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'function' object has no attribute 'named_parameters'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m----> 2\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43msince\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden_channels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1024\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_gnn_layers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m8\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_batch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1e-5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_loss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minference_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minference_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_wrapper\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/g_retriever.py:177\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model, dataset, checkpointing, tiny_llama)\u001b[0m\n\u001b[1;32m 174\u001b[0m model_save_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mllm\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 176\u001b[0m \u001b[38;5;66;03m# Step 3 Set Optimizer\u001b[39;00m\n\u001b[0;32m--> 177\u001b[0m params \u001b[38;5;241m=\u001b[39m [p \u001b[38;5;28;01mfor\u001b[39;00m _, p \u001b[38;5;129;01min\u001b[39;00m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnamed_parameters\u001b[49m() \u001b[38;5;28;01mif\u001b[39;00m p\u001b[38;5;241m.\u001b[39mrequires_grad]\n\u001b[1;32m 178\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdamW([\n\u001b[1;32m 179\u001b[0m {\n\u001b[1;32m 180\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparams\u001b[39m\u001b[38;5;124m'\u001b[39m: params,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 183\u001b[0m },\n\u001b[1;32m 184\u001b[0m ], betas\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0.9\u001b[39m, \u001b[38;5;241m0.95\u001b[39m))\n\u001b[1;32m 185\u001b[0m grad_steps \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'named_parameters'" + ] + } + ], + "source": [ + "start = time.time()\n", + "result = train(since=start, num_epochs=1,dataset=ds, hidden_channels=1024, num_gnn_layers=4, batch_size=8, eval_batch_size=16, lr=1e-5, loss_fn=get_loss, inference_fn=inference_wrapper, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@profileit(\"cuda\")\n", + "def profile_wrapper(model: torch.nn.Module, device_tensor: torch.Tensor, **kwargs):\n", + " kwargs['model'] = model\n", + " return train(**kwargs)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total Prep Time (prep_time) = 0.3\n", + "Total Prep Time (prep_time) = 1.93\n", "Training beginning...\n" ] }, @@ -74,18839 +121,20 @@ "name": "stderr", "output_type": "stream", "text": [ - "Epoch: 1|1: 0%| | 0/353 [00:00 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mProfiler\u001b[49m(model\u001b[38;5;241m=\u001b[39mmodel\u001b[38;5;241m.\u001b[39mgraph_encoder, use_cuda\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, profile_memory\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, max_depth\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m prof:\n\u001b[1;32m 2\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m 3\u001b[0m prep_time, dataset, gnn_llm_eval_outs \u001b[38;5;241m=\u001b[39m train(since\u001b[38;5;241m=\u001b[39mstart, num_epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, hidden_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1024\u001b[39m, num_gnn_layers\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m4\u001b[39m, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m8\u001b[39m, eval_batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m16\u001b[39m, lr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1e-5\u001b[39m, loss_fn\u001b[38;5;241m=\u001b[39mget_loss, inference_fn\u001b[38;5;241m=\u001b[39minference_step, dataset\u001b[38;5;241m=\u001b[39mds, model\u001b[38;5;241m=\u001b[39mmodel)\n", + "\u001b[0;31mNameError\u001b[0m: name 'Profiler' is not defined" + ] + } + ], + "source": [ + "with Profiler(model=model.graph_encoder, use_cuda=True, profile_memory=True, max_depth=1) as prof:\n", " start = time.time()\n", " prep_time, dataset, gnn_llm_eval_outs = train(since=start, num_epochs=1, hidden_channels=1024, num_gnn_layers=4, batch_size=8, eval_batch_size=16, lr=1e-5, loss_fn=get_loss, inference_fn=inference_step, dataset=ds, model=model)" ] @@ -24186,7 +258,7 @@ { "data": { "text/plain": [ - "('Module | Self CPU total | CPU total | Number of Calls\\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|-----------|----------------\\n-GAT | | | \\n-dropout | | | \\n--aten::dropout | 136.000us | 136.000us | 1413 \\n--cudaDeviceSynchronize | 4.750ms | 4.750ms | 1413 \\n-act | | | \\n--aten::relu | 280.710ms | 520.573ms | 1413 \\n--aten::clamp_min | 22.064ms | 239.863ms | 1413 \\n--cudaLaunchKernel | 217.799ms | 217.799ms | 1413 \\n--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 1413 \\n--cudaDeviceSynchronize | 3.419ms | 3.419ms | 1413 \\n-convs | | | \\n--0 | | | \\n---aggr_module | | | \\n----aten::view | 1.424ms | 1.424ms | 471 \\n----aten::expand_as | 615.000us | 1.819ms | 471 \\n----aten::expand | 980.000us | 1.365ms | 471 \\n----aten::as_strided | 440.000us | 440.000us | 1413 \\n----aten::new_zeros | 3.391ms | 199.119ms | 471 \\n----aten::new_empty | 4.615ms | 102.671ms | 471 \\n----aten::empty | 101.948ms | 101.948ms | 471 \\n----aten::zero_ | 1.214ms | 95.019ms | 471 \\n----aten::fill_ | 3.142ms | 93.805ms | 471 \\n----cudaLaunchKernel | 93.714ms | 93.714ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.842ms | 10.946ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.265ms | 1.265ms | 471 \\n---lin | | | \\n----aten::linear | 1.920ms | 230.667ms | 471 \\n----aten::t | 2.160ms | 3.650ms | 471 \\n----aten::transpose | 1.015ms | 1.490ms | 471 \\n----aten::as_strided | 475.000us | 475.000us | 471 \\n----aten::matmul | 1.212ms | 225.097ms | 471 \\n----aten::mm | 115.602ms | 223.885ms | 471 \\n----cudaFree | 5.856ms | 5.856ms | 2 \\n----cudaDeviceGetAttribute | 0.000us | 0.000us | 14 \\n----cudaGetSymbolAddress | 76.000us | 76.000us | 1 \\n----cudaMalloc | 183.000us | 183.000us | 3 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 984.000us | 984.000us | 241 \\n----cudaLaunchKernel | 92.051ms | 92.051ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.606ms | 1.606ms | 471 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 9.133ms | 9.133ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--1 | | | \\n---aggr_module | | | \\n----aten::view | 1.417ms | 1.417ms | 471 \\n----aten::expand_as | 556.000us | 1.754ms | 471 \\n----aten::expand | 976.000us | 1.364ms | 471 \\n----aten::as_strided | 444.000us | 444.000us | 1413 \\n----aten::new_zeros | 2.643ms | 181.493ms | 471 \\n----aten::new_empty | 2.877ms | 101.409ms | 471 \\n----aten::empty | 100.778ms | 100.778ms | 471 \\n----aten::zero_ | 1.140ms | 78.757ms | 471 \\n----aten::fill_ | 3.102ms | 77.617ms | 471 \\n----cudaLaunchKernel | 77.346ms | 77.346ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.741ms | 10.628ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.128ms | 1.128ms | 471 \\n---lin | | | \\n----aten::linear | 9.496ms | 194.940ms | 471 \\n----aten::t | 2.135ms | 3.325ms | 471 \\n----aten::transpose | 709.000us | 1.190ms | 471 \\n----aten::as_strided | 481.000us | 481.000us | 471 \\n----aten::matmul | 1.116ms | 190.137ms | 471 \\n----aten::mm | 104.854ms | 189.021ms | 471 \\n----cudaLaunchKernel | 75.774ms | 75.774ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.318ms | 1.318ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 208.000us | 208.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 8.185ms | 8.185ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--2 | | | \\n---aggr_module | | | \\n----aten::view | 1.422ms | 1.422ms | 471 \\n----aten::expand_as | 516.000us | 1.799ms | 471 \\n----aten::expand | 982.000us | 1.374ms | 471 \\n----aten::as_strided | 447.000us | 447.000us | 1413 \\n----aten::new_zeros | 4.523ms | 179.475ms | 471 \\n----aten::new_empty | 1.322ms | 110.879ms | 471 \\n----aten::empty | 110.271ms | 110.271ms | 471 \\n----aten::zero_ | 1.103ms | 67.282ms | 471 \\n----aten::fill_ | 3.111ms | 66.179ms | 471 \\n----cudaLaunchKernel | 65.813ms | 65.813ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.691ms | 10.490ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.046ms | 1.046ms | 471 \\n---lin | | | \\n----aten::linear | 7.860ms | 176.679ms | 471 \\n----aten::t | 2.104ms | 3.281ms | 471 \\n----aten::transpose | 700.000us | 1.177ms | 471 \\n----aten::as_strided | 477.000us | 477.000us | 471 \\n----aten::matmul | 1.124ms | 171.989ms | 471 \\n----aten::mm | 99.906ms | 170.865ms | 471 \\n----cudaLaunchKernel | 64.022ms | 64.022ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.434ms | 1.434ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 178.000us | 178.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 6.759ms | 6.759ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--3 | | | \\n---aggr_module | | | \\n----aten::view | 1.421ms | 1.421ms | 471 \\n----aten::expand_as | 539.000us | 1.889ms | 471 \\n----aten::expand | 1.020ms | 1.431ms | 471 \\n----aten::as_strided | 480.000us | 480.000us | 1413 \\n----aten::new_zeros | 3.339ms | 156.911ms | 471 \\n----aten::new_empty | 1.282ms | 93.703ms | 471 \\n----aten::empty | 93.156ms | 93.156ms | 471 \\n----aten::zero_ | 1.116ms | 61.915ms | 471 \\n----aten::fill_ | 3.116ms | 60.799ms | 471 \\n----cudaLaunchKernel | 60.432ms | 60.432ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.711ms | 10.529ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.044ms | 1.044ms | 471 \\n---lin | | | \\n----aten::linear | 10.282ms | 164.389ms | 471 \\n----aten::t | 2.049ms | 3.278ms | 471 \\n----aten::transpose | 753.000us | 1.229ms | 471 \\n----aten::as_strided | 476.000us | 476.000us | 471 \\n----aten::matmul | 1.070ms | 159.753ms | 471 \\n----aten::mm | 95.770ms | 158.683ms | 471 \\n----cudaStreamIsCapturing | 1.000us | 1.000us | 1 \\n----cudaMalloc | 152.000us | 152.000us | 1 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 1.023ms | 1.023ms | 511 \\n----cudaLaunchKernel | 32.548ms | 32.548ms | 471 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 172 \\n----cudaDeviceSynchronize | 6.656ms | 6.656ms | 471 \\n----cudaMemsetAsync | 29.189ms | 29.189ms | 230 \\n----Memset (Device) | 0.000us | 0.000us | 230 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 214 \\n----ampere_sgemm_128x32_sliced1x4_tn | 0.000us | 0.000us | 83 \\n----ampere_sgemm_128x64_tn | 0.000us | 0.000us | 2 \\n-norms | | | \\n--0 | | | \\n---module | | | \\n----aten::add_ | 79.108ms | 143.631ms | 353 \\n----cudaLaunchKernel | 74.679ms | 74.679ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 1.191ms | 84.025ms | 471 \\n----aten::_batch_norm_impl_index | 2.948ms | 82.488ms | 471 \\n----aten::empty | 30.383ms | 30.383ms | 1413 \\n----aten::native_batch_norm | 12.604ms | 52.167ms | 471 \\n----aten::empty_like | 750.000us | 3.136ms | 471 \\n----aten::empty_strided | 2.504ms | 2.504ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.285ms | 1.285ms | 471 \\n----aten::copy_ | 747.000us | 23.627ms | 118 \\n----cudaMemcpyAsync | 22.880ms | 22.880ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--1 | | | \\n---module | | | \\n----aten::add_ | 87.275ms | 139.715ms | 353 \\n----cudaLaunchKernel | 59.714ms | 59.714ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 528.000us | 77.774ms | 471 \\n----aten::_batch_norm_impl_index | 3.016ms | 76.477ms | 471 \\n----aten::empty | 32.189ms | 32.189ms | 1413 \\n----aten::native_batch_norm | 11.195ms | 44.783ms | 471 \\n----aten::empty_like | 634.000us | 3.093ms | 471 \\n----aten::empty_strided | 2.520ms | 2.520ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.066ms | 1.066ms | 471 \\n----aten::copy_ | 745.000us | 20.607ms | 118 \\n----cudaMemcpyAsync | 19.862ms | 19.862ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--2 | | | \\n---module | | | \\n-- -aten::add_ | 86.501ms | 132.128ms | 353 \\n-- -cudaLaunchKernel | 52.722ms | 52.722ms | 1648 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n-- -aten::batch_norm | 873.000us | 72.646ms | 471 \\n-- -aten::_batch_norm_impl_index | 2.720ms | 71.364ms | 471 \\n-- -aten::empty | 30.893ms | 30.893ms | 1413 \\n-- -aten::native_batch_norm | 10.998ms | 40.990ms | 471 \\n-- -aten::empty_like | 632.000us | 3.084ms | 471 \\n-- -aten::empty_strided | 2.510ms | 2.510ms | 471 \\n-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n-- -cudaDeviceSynchronize | 1.048ms | 1.048ms | 471 \\n-- -aten::copy_ | 745.000us | 17.173ms | 118 \\n-- -cudaMemcpyAsync | 16.428ms | 16.428ms | 118 \\n-- -Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--3 | | | \\n-_trim | | | \\n',\n", + "('Module | Self CPU total | CPU total | Number of Calls\\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|-----------|----------------\\n-GAT | | | \\n-dropout | | | \\n--aten::dropout | 142.000us | 142.000us | 1413 \\n--cudaDeviceSynchronize | 5.410ms | 5.410ms | 1413 \\n-act | | | \\n--aten::relu | 252.233ms | 476.470ms | 1413 \\n--aten::clamp_min | 23.247ms | 224.237ms | 1413 \\n--cudaLaunchKernel | 200.990ms | 200.990ms | 1413 \\n--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 1413 \\n--cudaDeviceSynchronize | 3.242ms | 3.242ms | 1413 \\n-convs | | | \\n--0 | | | \\n---aggr_module | | | \\n----aten::view | 1.461ms | 1.461ms | 471 \\n----aten::expand_as | 589.000us | 1.912ms | 471 \\n----aten::expand | 1.001ms | 1.367ms | 471 \\n----aten::as_strided | 428.000us | 428.000us | 1413 \\n----aten::new_zeros | 3.599ms | 180.361ms | 471 \\n----aten::new_empty | 2.758ms | 93.925ms | 471 \\n----aten::empty | 93.399ms | 93.399ms | 471 \\n----aten::zero_ | 1.110ms | 84.988ms | 471 \\n----aten::fill_ | 3.016ms | 83.878ms | 471 \\n----cudaLaunchKernel | 83.662ms | 83.662ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.865ms | 10.726ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.188ms | 1.188ms | 471 \\n---lin | | | \\n----aten::linear | 1.987ms | 211.347ms | 471 \\n----aten::t | 2.331ms | 3.810ms | 471 \\n----aten::transpose | 1.002ms | 1.478ms | 471 \\n----aten::as_strided | 477.000us | 477.000us | 471 \\n----aten::matmul | 1.079ms | 205.550ms | 471 \\n----aten::mm | 107.088ms | 204.471ms | 471 \\n----cudaFree | 5.830ms | 5.830ms | 2 \\n----cudaDeviceGetAttribute | 0.000us | 0.000us | 14 \\n----cudaGetSymbolAddress | 71.000us | 71.000us | 1 \\n----cudaMalloc | 171.000us | 171.000us | 3 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 984.000us | 984.000us | 241 \\n----cudaLaunchKernel | 80.716ms | 80.716ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.609ms | 1.609ms | 471 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 9.611ms | 9.611ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--1 | | | \\n---aggr_module | | | \\n----aten::view | 1.464ms | 1.464ms | 471 \\n----aten::expand_as | 582.000us | 1.875ms | 471 \\n----aten::expand | 988.000us | 1.353ms | 471 \\n----aten::as_strided | 429.000us | 429.000us | 1413 \\n----aten::new_zeros | 2.802ms | 171.042ms | 471 \\n----aten::new_empty | 3.360ms | 98.481ms | 471 \\n----aten::empty | 98.016ms | 98.016ms | 471 \\n----aten::zero_ | 1.045ms | 71.202ms | 471 \\n----aten::fill_ | 2.970ms | 70.157ms | 471 \\n----cudaLaunchKernel | 69.697ms | 69.697ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.770ms | 10.343ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.054ms | 1.054ms | 471 \\n---lin | | | \\n----aten::linear | 6.509ms | 182.224ms | 471 \\n----aten::t | 2.271ms | 3.509ms | 471 \\n----aten::transpose | 760.000us | 1.238ms | 471 \\n----aten::as_strided | 478.000us | 478.000us | 471 \\n----aten::matmul | 967.000us | 177.245ms | 471 \\n----aten::mm | 99.358ms | 176.278ms | 471 \\n----cudaLaunchKernel | 68.871ms | 68.871ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.225ms | 1.225ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 196.000us | 196.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 7.853ms | 7.853ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--2 | | | \\n---aggr_module | | | \\n----aten::view | 1.469ms | 1.469ms | 471 \\n----aten::expand_as | 584.000us | 1.927ms | 471 \\n----aten::expand | 1.040ms | 1.424ms | 471 \\n----aten::as_strided | 461.000us | 461.000us | 1413 \\n----aten::new_zeros | 2.053ms | 164.227ms | 471 \\n----aten::new_empty | 3.461ms | 100.500ms | 471 \\n----aten::empty | 100.034ms | 100.034ms | 471 \\n----aten::zero_ | 1.031ms | 62.406ms | 471 \\n----aten::fill_ | 2.997ms | 61.375ms | 471 \\n----cudaLaunchKernel | 60.833ms | 60.833ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.764ms | 10.296ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.017ms | 1.017ms | 471 \\n---lin | | | \\n----aten::linear | 6.985ms | 167.387ms | 471 \\n----aten::t | 2.268ms | 3.526ms | 471 \\n----aten::transpose | 784.000us | 1.258ms | 471 \\n----aten::as_strided | 474.000us | 474.000us | 471 \\n----aten::matmul | 979.000us | 162.455ms | 471 \\n----aten::mm | 95.898ms | 161.476ms | 471 \\n----cudaLaunchKernel | 59.421ms | 59.421ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.170ms | 1.170ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 186.000us | 186.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 5.971ms | 5.971ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--3 | | | \\n---aggr_module | | | \\n----aten::view | 1.455ms | 1.455ms | 471 \\n----aten::expand_as | 626.000us | 1.932ms | 471 \\n----aten::expand | 1.094ms | 1.507ms | 471 \\n----aten::as_strided | 508.000us | 508.000us | 1413 \\n----aten::new_zeros | 2.890ms | 149.597ms | 471 \\n----aten::new_empty | 2.133ms | 89.208ms | 471 \\n----aten::empty | 88.735ms | 88.735ms | 471 \\n----aten::zero_ | 1.005ms | 59.082ms | 471 \\n----aten::fill_ | 2.999ms | 58.077ms | 471 \\n----cudaLaunchKernel | 57.526ms | 57.526ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.812ms | 10.355ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.048ms | 1.048ms | 471 \\n---lin | | | \\n----aten::linear | 6.715ms | 157.757ms | 471 \\n----aten::t | 2.212ms | 3.499ms | 471 \\n----aten::transpose | 813.000us | 1.287ms | 471 \\n----aten::as_strided | 474.000us | 474.000us | 471 \\n----aten::matmul | 959.000us | 152.888ms | 471 \\n----aten::mm | 92.151ms | 151.929ms | 471 \\n----cudaStreamIsCapturing | 1.000us | 1.000us | 1 \\n----cudaMalloc | 145.000us | 145.000us | 1 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 1.077ms | 1.077ms | 511 \\n----cudaLaunchKernel | 30.929ms | 30.929ms | 471 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 172 \\n----cudaDeviceSynchronize | 6.041ms | 6.041ms | 471 \\n----cudaMemsetAsync | 27.626ms | 27.626ms | 230 \\n----Memset (Device) | 0.000us | 0.000us | 230 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 214 \\n----ampere_sgemm_128x32_sliced1x4_tn | 0.000us | 0.000us | 83 \\n----ampere_sgemm_128x64_tn | 0.000us | 0.000us | 2 \\n-norms | | | \\n--0 | | | \\n---module | | | \\n----[memory] | 0.000us | 0.000us | 744 \\n----aten::add_ | 70.478ms | 126.577ms | 353 \\n----cudaLaunchKernel | 65.908ms | 65.908ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | -532.000us | 82.041ms | 471 \\n----aten::_batch_norm_impl_index | 4.488ms | 80.580ms | 471 \\n----aten::empty | 28.837ms | 28.837ms | 1413 \\n----aten::native_batch_norm | 12.558ms | 51.947ms | 471 \\n----aten::empty_like | 722.000us | 3.170ms | 471 \\n----aten::empty_strided | 2.523ms | 2.523ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.191ms | 1.191ms | 471 \\n----aten::copy_ | 781.000us | 23.765ms | 118 \\n----cudaMemcpyAsync | 22.984ms | 22.984ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--1 | | | \\n---module | | | \\n----[memory] | 0.000us | 0.000us | 739 \\n----aten::add_ | 81.224ms | 129.447ms | 353 \\n----cudaLaunchKernel | 55.220ms | 55.220ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 661.000us | 72.788ms | 471 \\n----aten::_batch_norm_impl_index | 2.810ms | 71.489ms | 471 \\n----aten::empty | 29.551ms | 29.551ms | 1413 \\n----aten::native_batch_norm | 11.271ms | 42.637ms | 471 \\n----aten::empty_like | 667.000us | 3.107ms | 471 \\n----aten::empty_strided | 2.492ms | 2.492ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.099ms | 1.099ms | 471 \\n----aten::copy_ | 776.000us | 18.528ms | 118 \\n----cudaMemcpyAsync | 17.752ms | 17.752ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--2 | | | \\n---module | | | \\n-- -[memory] | 0.000us | 0.000us | 741 \\n-- -aten::add_ | 79.923ms | 123.217ms | 353 \\n-- -cudaLaunchKernel | 50.052ms | 50.052ms | 1648 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n-- -aten::batch_norm | 1.028ms | 69.357ms | 471 \\n-- -aten::_batch_norm_impl_index | 2.529ms | 68.094ms | 471 \\n-- -aten::empty | 28.910ms | 28.910ms | 1413 \\n-- -aten::native_batch_norm | 11.237ms | 39.924ms | 471 \\n-- -aten::empty_like | 659.000us | 3.080ms | 471 \\n-- -aten::empty_strided | 2.473ms | 2.473ms | 471 \\n-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n-- -cudaDeviceSynchronize | 1.042ms | 1.042ms | 471 \\n-- -aten::copy_ | 777.000us | 16.076ms | 118 \\n-- -cudaMemcpyAsync | 15.299ms | 15.299ms | 118 \\n-- -Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--3 | | | \\n-_trim | | | \\n',\n", " ['Module',\n", " 'Self CPU total',\n", " 'CPU total',\n", @@ -24197,11 +269,19 @@ " 'Self CUDA Mem',\n", " 'CUDA Mem',\n", " 'Number of Calls'],\n", - " {'aten::dropout': [136, 136, 0, 0, 0, 0, 0, 0, 1413],\n", - " 'cudaDeviceSynchronize': [1048, 1048, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::relu': [280710, 520573, 0, 1413, 0, 0, 0, 0, 1413],\n", - " 'aten::clamp_min': [22064, 239863, 1413, 1413, 0, 0, 0, 0, 1413],\n", - " 'cudaLaunchKernel': [52722, 52722, 0, 0, 0, 0, 0, 0, 1648],\n", + " {'aten::dropout': [142, 142, 0, 0, 0, 0, 0, 0, 1413],\n", + " 'cudaDeviceSynchronize': [1042, 1042, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::relu': [252233, 476470, 0, 1413, 0, 0, 0, 460271616, 1413],\n", + " 'aten::clamp_min': [23247,\n", + " 224237,\n", + " 1413,\n", + " 1413,\n", + " 0,\n", + " 0,\n", + " 460271616,\n", + " 460271616,\n", + " 1413],\n", + " 'cudaLaunchKernel': [50052, 50052, 0, 0, 0, 0, 0, 0, 1648],\n", " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", " 0,\n", " 1413,\n", @@ -24211,45 +291,45 @@ " 0,\n", " 0,\n", " 1413],\n", - " 'aten::view': [1421, 1421, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::expand_as': [539, 1889, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::expand': [1020, 1431, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::as_strided': [476, 476, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::new_zeros': [3339, 156911, 0, 367, 0, 0, 0, 0, 471],\n", - " 'aten::new_empty': [1282, 93703, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::empty': [30893, 30893, 0, 0, 0, 0, 0, 0, 1413],\n", - " 'aten::zero_': [1116, 61915, 0, 381, 0, 0, 0, 0, 471],\n", - " 'aten::fill_': [3116, 60799, 381, 381, 0, 0, 0, 0, 471],\n", + " 'aten::view': [1455, 1455, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::expand_as': [626, 1932, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::expand': [1094, 1507, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::as_strided': [474, 474, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::new_zeros': [2890, 149597, 0, 334, 0, 0, -10797056, 731824128, 471],\n", + " 'aten::new_empty': [2133, 89208, 0, 0, 0, 0, 10797056, 731824128, 471],\n", + " 'aten::empty': [28910, 28910, 0, 0, 0, 0, 3858432, 3858432, 1413],\n", + " 'aten::zero_': [1005, 59082, 0, 346, 0, 0, 0, 0, 471],\n", + " 'aten::fill_': [2999, 58077, 346, 346, 0, 0, 0, 0, 471],\n", " 'void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)': [0,\n", " 0,\n", - " 381,\n", - " 381,\n", + " 346,\n", + " 346,\n", " 0,\n", " 0,\n", " 0,\n", " 0,\n", " 471],\n", - " 'aten::scatter_add_': [7711, 10529, 1307, 1307, 0, 0, 0, 0, 471],\n", + " 'aten::scatter_add_': [7812, 10355, 1277, 1277, 0, 0, 0, 0, 471],\n", " 'void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})': [0,\n", " 0,\n", - " 1307,\n", - " 1307,\n", + " 1277,\n", + " 1277,\n", " 0,\n", " 0,\n", " 0,\n", " 0,\n", " 471],\n", - " 'aten::linear': [10282, 164389, 0, 16237, 0, 0, 0, 0, 471],\n", - " 'aten::t': [2049, 3278, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::transpose': [753, 1229, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::matmul': [1070, 159753, 0, 17112, 0, 0, 0, 0, 471],\n", - " 'aten::mm': [95770, 158683, 17112, 17112, 0, 0, 0, 0, 471],\n", - " 'cudaFree': [5856, 5856, 0, 0, 0, 0, 0, 0, 2],\n", + " 'aten::linear': [6715, 157757, 0, 16142, 0, 0, 21594112, 781910016, 471],\n", + " 'aten::t': [2212, 3499, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::transpose': [813, 1287, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::matmul': [959, 152888, 0, 16628, 0, 0, 0, 781910016, 471],\n", + " 'aten::mm': [92151, 151929, 16628, 16628, 0, 0, 781910016, 781910016, 471],\n", + " 'cudaFree': [5830, 5830, 0, 0, 0, 0, 0, 0, 2],\n", " 'cudaDeviceGetAttribute': [0, 0, 0, 0, 0, 0, 0, 0, 14],\n", - " 'cudaGetSymbolAddress': [76, 76, 0, 0, 0, 0, 0, 0, 1],\n", - " 'cudaMalloc': [152, 152, 0, 0, 0, 0, 0, 0, 1],\n", - " 'cudaOccupancyMaxActiveBlocksPerMultiprocessor': [1023,\n", - " 1023,\n", + " 'cudaGetSymbolAddress': [71, 71, 0, 0, 0, 0, 0, 0, 1],\n", + " 'cudaMalloc': [145, 145, 0, 0, 0, 0, 0, 0, 1],\n", + " 'cudaOccupancyMaxActiveBlocksPerMultiprocessor': [1077,\n", + " 1077,\n", " 0,\n", " 0,\n", " 0,\n", @@ -24257,24 +337,25 @@ " 0,\n", " 0,\n", " 511],\n", - " 'ampere_sgemm_128x32_tn': [0, 0, 9892, 9892, 0, 0, 0, 0, 214],\n", + " 'ampere_sgemm_128x32_tn': [0, 0, 9588, 9588, 0, 0, 0, 0, 214],\n", " 'void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)': [0,\n", " 0,\n", - " 456,\n", - " 456,\n", + " 434,\n", + " 434,\n", " 0,\n", " 0,\n", " 0,\n", " 0,\n", " 425],\n", - " 'ampere_sgemm_64x32_sliced1x4_tn': [0, 0, 1482, 1482, 0, 0, 0, 0, 132],\n", - " 'ampere_sgemm_32x32_sliced1x4_tn': [0, 0, 4233, 4233, 0, 0, 0, 0, 172],\n", - " 'cudaMemsetAsync': [29189, 29189, 0, 0, 0, 0, 0, 0, 230],\n", + " 'ampere_sgemm_64x32_sliced1x4_tn': [0, 0, 1419, 1419, 0, 0, 0, 0, 132],\n", + " 'ampere_sgemm_32x32_sliced1x4_tn': [0, 0, 4197, 4197, 0, 0, 0, 0, 172],\n", + " 'cudaMemsetAsync': [27626, 27626, 0, 0, 0, 0, 0, 0, 230],\n", " 'Memset (Device)': [0, 0, 0, 0, 0, 0, 0, 0, 230],\n", " 'cudaStreamIsCapturing': [1, 1, 0, 0, 0, 0, 0, 0, 1],\n", - " 'ampere_sgemm_128x32_sliced1x4_tn': [0, 0, 2876, 2876, 0, 0, 0, 0, 83],\n", - " 'ampere_sgemm_128x64_tn': [0, 0, 111, 111, 0, 0, 0, 0, 2],\n", - " 'aten::add_': [86501, 132128, 353, 353, 0, 0, 0, 0, 353],\n", + " 'ampere_sgemm_128x32_sliced1x4_tn': [0, 0, 2739, 2739, 0, 0, 0, 0, 83],\n", + " 'ampere_sgemm_128x64_tn': [0, 0, 104, 104, 0, 0, 0, 0, 2],\n", + " '[memory]': [0, 0, 0, 0, 0, 0, -143360, -143360, 741],\n", + " 'aten::add_': [79923, 123217, 353, 353, 0, 0, 0, 0, 353],\n", " 'void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)': [0,\n", " 0,\n", " 353,\n", @@ -24284,15 +365,31 @@ " 0,\n", " 0,\n", " 353],\n", - " 'aten::batch_norm': [873, 72646, 0, 2573, 0, 0, 0, 0, 471],\n", - " 'aten::_batch_norm_impl_index': [2720, 71364, 0, 2539, 0, 0, 0, 0, 471],\n", - " 'aten::native_batch_norm': [10998, 40990, 2479, 2597, 0, 0, 0, 0, 471],\n", - " 'aten::empty_like': [632, 3084, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::empty_strided': [2510, 2510, 0, 0, 0, 0, 0, 0, 471],\n", + " 'aten::batch_norm': [1028, 69357, 0, 2562, 0, 0, -1417216, 156459008, 471],\n", + " 'aten::_batch_norm_impl_index': [2529,\n", + " 68094,\n", + " 0,\n", + " 2560,\n", + " 0,\n", + " 0,\n", + " 1871872,\n", + " 157282304,\n", + " 471],\n", + " 'aten::native_batch_norm': [11237,\n", + " 39924,\n", + " 2474,\n", + " 2592,\n", + " 0,\n", + " 0,\n", + " -3280896,\n", + " 157282304,\n", + " 471],\n", + " 'aten::empty_like': [659, 3080, 0, 0, 0, 0, 3280896, 153423872, 471],\n", + " 'aten::empty_strided': [2473, 2473, 0, 0, 0, 0, 153423872, 153423872, 471],\n", " 'void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)': [0,\n", " 0,\n", - " 1066,\n", - " 1066,\n", + " 1061,\n", + " 1061,\n", " 0,\n", " 0,\n", " 0,\n", @@ -24316,8 +413,8 @@ " 0,\n", " 0,\n", " 471],\n", - " 'aten::copy_': [745, 17173, 118, 118, 0, 0, 0, 0, 118],\n", - " 'cudaMemcpyAsync': [16428, 16428, 0, 0, 0, 0, 0, 0, 118],\n", + " 'aten::copy_': [777, 16076, 118, 118, 0, 0, 0, 0, 118],\n", + " 'cudaMemcpyAsync': [15299, 15299, 0, 0, 0, 0, 0, 0, 118],\n", " 'Memcpy DtoD (Device -> Device)': [0, 0, 118, 118, 0, 0, 0, 0, 118],\n", " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", " 0,\n", @@ -24457,6 +554,7 @@ " '-convs--3---lin----ampere_sgemm_128x32_tn',\n", " '-convs--3---lin----ampere_sgemm_128x32_sliced1x4_tn',\n", " '-convs--3---lin----ampere_sgemm_128x64_tn',\n", + " '-norms--0---module----[memory]',\n", " '-norms--0---module----aten::add_',\n", " '-norms--0---module----cudaLaunchKernel',\n", " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", @@ -24474,6 +572,7 @@ " '-norms--0---module----cudaMemcpyAsync',\n", " '-norms--0---module----Memcpy DtoD (Device -> Device)',\n", " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", + " '-norms--1---module----[memory]',\n", " '-norms--1---module----aten::add_',\n", " '-norms--1---module----cudaLaunchKernel',\n", " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", @@ -24491,6 +590,7 @@ " '-norms--1---module----cudaMemcpyAsync',\n", " '-norms--1---module----Memcpy DtoD (Device -> Device)',\n", " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", + " '-norms-- -[memory]',\n", " '-norms-- -aten::add_',\n", " '-norms-- -cudaLaunchKernel',\n", " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", @@ -24508,8 +608,8 @@ " '-norms-- -cudaMemcpyAsync',\n", " '-norms-- -Memcpy DtoD (Device -> Device)',\n", " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)'],\n", - " [['136.000us',\n", - " '136.000us',\n", + " [['142.000us',\n", + " '142.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24517,8 +617,8 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['4.750ms',\n", - " '4.750ms',\n", + " ['5.410ms',\n", + " '5.410ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24526,26 +626,26 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['280.710ms',\n", - " '520.573ms',\n", + " ['252.233ms',\n", + " '476.470ms',\n", " '0.000us',\n", " '1.413ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", + " '438.95 Mb',\n", " '1413'],\n", - " ['22.064ms',\n", - " '239.863ms',\n", + " ['23.247ms',\n", + " '224.237ms',\n", " '1.413ms',\n", " '1.413ms',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '438.95 Mb',\n", + " '438.95 Mb',\n", " '1413'],\n", - " ['217.799ms',\n", - " '217.799ms',\n", + " ['200.990ms',\n", + " '200.990ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24562,8 +662,8 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['3.419ms',\n", - " '3.419ms',\n", + " ['3.242ms',\n", + " '3.242ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24571,8 +671,8 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['1.424ms',\n", - " '1.424ms',\n", + " ['1.461ms',\n", + " '1.461ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24580,8 +680,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['615.000us',\n", - " '1.819ms',\n", + " ['589.000us',\n", + " '1.912ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24589,8 +689,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['980.000us',\n", - " '1.365ms',\n", + " ['1.001ms',\n", + " '1.367ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24598,8 +698,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['440.000us',\n", - " '440.000us',\n", + " ['428.000us',\n", + " '428.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24607,53 +707,53 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['3.391ms',\n", - " '199.119ms',\n", + " ['3.599ms',\n", + " '180.361ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '35.000us',\n", - " '0 b',\n", - " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '-3.38 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['4.615ms',\n", - " '102.671ms',\n", + " ['2.758ms',\n", + " '93.925ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.38 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['101.948ms',\n", - " '101.948ms',\n", + " ['93.399ms',\n", + " '93.399ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['1.214ms',\n", - " '95.019ms',\n", + " ['1.110ms',\n", + " '84.988ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '39.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['3.142ms',\n", - " '93.805ms',\n", - " '39.000us',\n", - " '39.000us',\n", + " ['3.016ms',\n", + " '83.878ms',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['93.714ms',\n", - " '93.714ms',\n", + " ['83.662ms',\n", + " '83.662ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24663,15 +763,15 @@ " '942'],\n", " ['0.000us',\n", " '0.000us',\n", - " '39.000us',\n", - " '39.000us',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['7.842ms',\n", - " '10.946ms',\n", + " ['7.865ms',\n", + " '10.726ms',\n", " '942.000us',\n", " '942.000us',\n", " '0 b',\n", @@ -24688,8 +788,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.265ms',\n", - " '1.265ms',\n", + " ['1.188ms',\n", + " '1.188ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24697,17 +797,17 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.920ms',\n", - " '230.667ms',\n", + " ['1.987ms',\n", + " '211.347ms',\n", " '0.000us',\n", - " '6.433ms',\n", - " '0 b',\n", + " '6.005ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '154.44 Mb',\n", " '471'],\n", - " ['2.160ms',\n", - " '3.650ms',\n", + " ['2.331ms',\n", + " '3.810ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24715,8 +815,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.015ms',\n", - " '1.490ms',\n", + " ['1.002ms',\n", + " '1.478ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24724,8 +824,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['475.000us',\n", - " '475.000us',\n", + " ['477.000us',\n", + " '477.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24733,26 +833,26 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.212ms',\n", - " '225.097ms',\n", + " ['1.079ms',\n", + " '205.550ms',\n", " '0.000us',\n", - " '6.433ms',\n", - " '0 b',\n", + " '6.005ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '154.44 Mb',\n", " '471'],\n", - " ['115.602ms',\n", - " '223.885ms',\n", - " '6.433ms',\n", - " '6.433ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['107.088ms',\n", + " '204.471ms',\n", + " '6.005ms',\n", + " '6.005ms',\n", " '0 b',\n", " '0 b',\n", + " '154.44 Mb',\n", + " '154.44 Mb',\n", " '471'],\n", - " ['5.856ms',\n", - " '5.856ms',\n", + " ['5.830ms',\n", + " '5.830ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24769,8 +869,8 @@ " '0 b',\n", " '0 b',\n", " '14'],\n", - " ['76.000us',\n", - " '76.000us',\n", + " ['71.000us',\n", + " '71.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24778,8 +878,8 @@ " '0 b',\n", " '0 b',\n", " '1'],\n", - " ['183.000us',\n", - " '183.000us',\n", + " ['171.000us',\n", + " '171.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24796,8 +896,8 @@ " '0 b',\n", " '0 b',\n", " '241'],\n", - " ['92.051ms',\n", - " '92.051ms',\n", + " ['80.716ms',\n", + " '80.716ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24807,8 +907,8 @@ " '896'],\n", " ['0.000us',\n", " '0.000us',\n", - " '839.000us',\n", - " '839.000us',\n", + " '813.000us',\n", + " '813.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -24816,15 +916,15 @@ " '98'],\n", " ['0.000us',\n", " '0.000us',\n", - " '511.000us',\n", - " '511.000us',\n", + " '500.000us',\n", + " '500.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '425'],\n", - " ['1.606ms',\n", - " '1.606ms',\n", + " ['1.609ms',\n", + " '1.609ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24834,8 +934,8 @@ " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.767ms',\n", - " '1.767ms',\n", + " '1.605ms',\n", + " '1.605ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -24843,15 +943,15 @@ " '132'],\n", " ['0.000us',\n", " '0.000us',\n", - " '3.316ms',\n", - " '3.316ms',\n", + " '3.087ms',\n", + " '3.087ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '241'],\n", - " ['9.133ms',\n", - " '9.133ms',\n", + " ['9.611ms',\n", + " '9.611ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24868,8 +968,8 @@ " '0 b',\n", " '0 b',\n", " '44'],\n", - " ['1.417ms',\n", - " '1.417ms',\n", + " ['1.464ms',\n", + " '1.464ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24877,8 +977,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['556.000us',\n", - " '1.754ms',\n", + " ['582.000us',\n", + " '1.875ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24886,8 +986,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['976.000us',\n", - " '1.364ms',\n", + " ['988.000us',\n", + " '1.353ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24895,8 +995,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['444.000us',\n", - " '444.000us',\n", + " ['429.000us',\n", + " '429.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24904,53 +1004,53 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['2.643ms',\n", - " '181.493ms',\n", + " ['2.802ms',\n", + " '171.042ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '3.000us',\n", - " '0 b',\n", - " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '-4.36 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.877ms',\n", - " '101.409ms',\n", + " ['3.360ms',\n", + " '98.481ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '4.36 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['100.778ms',\n", - " '100.778ms',\n", + " ['98.016ms',\n", + " '98.016ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['1.140ms',\n", - " '78.757ms',\n", + " ['1.045ms',\n", + " '71.202ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '4.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['3.102ms',\n", - " '77.617ms',\n", - " '4.000us',\n", - " '4.000us',\n", + " ['2.970ms',\n", + " '70.157ms',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['77.346ms',\n", - " '77.346ms',\n", + " ['69.697ms',\n", + " '69.697ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24960,15 +1060,15 @@ " '942'],\n", " ['0.000us',\n", " '0.000us',\n", - " '4.000us',\n", - " '4.000us',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['7.741ms',\n", - " '10.628ms',\n", + " ['7.770ms',\n", + " '10.343ms',\n", " '942.000us',\n", " '942.000us',\n", " '0 b',\n", @@ -24985,8 +1085,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.128ms',\n", - " '1.128ms',\n", + " ['1.054ms',\n", + " '1.054ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -24994,17 +1094,17 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['9.496ms',\n", - " '194.940ms',\n", + " ['6.509ms',\n", + " '182.224ms',\n", " '0.000us',\n", - " '5.475ms',\n", - " '0 b',\n", - " '0 b',\n", + " '5.265ms',\n", " '0 b',\n", " '0 b',\n", + " '3.70 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.135ms',\n", - " '3.325ms',\n", + " ['2.271ms',\n", + " '3.509ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25012,8 +1112,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['709.000us',\n", - " '1.190ms',\n", + " ['760.000us',\n", + " '1.238ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25021,8 +1121,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['481.000us',\n", - " '481.000us',\n", + " ['478.000us',\n", + " '478.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25030,26 +1130,26 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.116ms',\n", - " '190.137ms',\n", + " ['967.000us',\n", + " '177.245ms',\n", " '0.000us',\n", - " '5.671ms',\n", - " '0 b',\n", + " '5.410ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['104.854ms',\n", - " '189.021ms',\n", - " '5.671ms',\n", - " '5.671ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['99.358ms',\n", + " '176.278ms',\n", + " '5.410ms',\n", + " '5.410ms',\n", " '0 b',\n", " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['75.774ms',\n", - " '75.774ms',\n", + " ['68.871ms',\n", + " '68.871ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25059,8 +1159,8 @@ " '896'],\n", " ['0.000us',\n", " '0.000us',\n", - " '692.000us',\n", - " '692.000us',\n", + " '688.000us',\n", + " '688.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25068,15 +1168,15 @@ " '98'],\n", " ['0.000us',\n", " '0.000us',\n", - " '459.000us',\n", - " '459.000us',\n", + " '434.000us',\n", + " '434.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '425'],\n", - " ['1.318ms',\n", - " '1.318ms',\n", + " ['1.225ms',\n", + " '1.225ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25084,8 +1184,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['208.000us',\n", - " '208.000us',\n", + " ['196.000us',\n", + " '196.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25095,8 +1195,8 @@ " '132'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.481ms',\n", - " '1.481ms',\n", + " '1.407ms',\n", + " '1.407ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25104,15 +1204,15 @@ " '132'],\n", " ['0.000us',\n", " '0.000us',\n", - " '3.039ms',\n", - " '3.039ms',\n", + " '2.881ms',\n", + " '2.881ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '241'],\n", - " ['8.185ms',\n", - " '8.185ms',\n", + " ['7.853ms',\n", + " '7.853ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25129,8 +1229,8 @@ " '0 b',\n", " '0 b',\n", " '44'],\n", - " ['1.422ms',\n", - " '1.422ms',\n", + " ['1.469ms',\n", + " '1.469ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25138,8 +1238,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['516.000us',\n", - " '1.799ms',\n", + " ['584.000us',\n", + " '1.927ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25147,8 +1247,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['982.000us',\n", - " '1.374ms',\n", + " ['1.040ms',\n", + " '1.424ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25156,8 +1256,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['447.000us',\n", - " '447.000us',\n", + " ['461.000us',\n", + " '461.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25165,53 +1265,53 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['4.523ms',\n", - " '179.475ms',\n", + " ['2.053ms',\n", + " '164.227ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '4.000us',\n", - " '0 b',\n", - " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '-4.03 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['1.322ms',\n", - " '110.879ms',\n", + " ['3.461ms',\n", + " '100.500ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '4.03 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['110.271ms',\n", - " '110.271ms',\n", + " ['100.034ms',\n", + " '100.034ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['1.103ms',\n", - " '67.282ms',\n", + " ['1.031ms',\n", + " '62.406ms',\n", + " '0.000us',\n", " '0.000us',\n", - " '4.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['3.111ms',\n", - " '66.179ms',\n", - " '4.000us',\n", - " '4.000us',\n", + " ['2.997ms',\n", + " '61.375ms',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['65.813ms',\n", - " '65.813ms',\n", + " ['60.833ms',\n", + " '60.833ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25221,15 +1321,15 @@ " '942'],\n", " ['0.000us',\n", " '0.000us',\n", - " '4.000us',\n", - " '4.000us',\n", + " '0.000us',\n", + " '0.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['7.691ms',\n", - " '10.490ms',\n", + " ['7.764ms',\n", + " '10.296ms',\n", " '942.000us',\n", " '942.000us',\n", " '0 b',\n", @@ -25246,8 +1346,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.046ms',\n", - " '1.046ms',\n", + " ['1.017ms',\n", + " '1.017ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25255,17 +1355,17 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['7.860ms',\n", - " '176.679ms',\n", + " ['6.985ms',\n", + " '167.387ms',\n", " '0.000us',\n", - " '5.494ms',\n", - " '0 b',\n", - " '0 b',\n", + " '5.307ms',\n", " '0 b',\n", " '0 b',\n", + " '4.06 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.104ms',\n", - " '3.281ms',\n", + " ['2.268ms',\n", + " '3.526ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25273,8 +1373,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['700.000us',\n", - " '1.177ms',\n", + " ['784.000us',\n", + " '1.258ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25282,8 +1382,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['477.000us',\n", - " '477.000us',\n", + " ['474.000us',\n", + " '474.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25291,26 +1391,26 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.124ms',\n", - " '171.989ms',\n", + " ['979.000us',\n", + " '162.455ms',\n", " '0.000us',\n", - " '5.675ms',\n", - " '0 b',\n", + " '5.470ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['99.906ms',\n", - " '170.865ms',\n", - " '5.675ms',\n", - " '5.675ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['95.898ms',\n", + " '161.476ms',\n", + " '5.470ms',\n", + " '5.470ms',\n", " '0 b',\n", " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['64.022ms',\n", - " '64.022ms',\n", + " ['59.421ms',\n", + " '59.421ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25320,8 +1420,8 @@ " '896'],\n", " ['0.000us',\n", " '0.000us',\n", - " '696.000us',\n", - " '696.000us',\n", + " '688.000us',\n", + " '688.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25329,15 +1429,15 @@ " '98'],\n", " ['0.000us',\n", " '0.000us',\n", - " '456.000us',\n", - " '456.000us',\n", + " '434.000us',\n", + " '434.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '425'],\n", - " ['1.434ms',\n", - " '1.434ms',\n", + " ['1.170ms',\n", + " '1.170ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25345,8 +1445,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['178.000us',\n", - " '178.000us',\n", + " ['186.000us',\n", + " '186.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25356,8 +1456,8 @@ " '132'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.482ms',\n", - " '1.482ms',\n", + " '1.419ms',\n", + " '1.419ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25365,15 +1465,15 @@ " '132'],\n", " ['0.000us',\n", " '0.000us',\n", - " '3.041ms',\n", - " '3.041ms',\n", + " '2.929ms',\n", + " '2.929ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '241'],\n", - " ['6.759ms',\n", - " '6.759ms',\n", + " ['5.971ms',\n", + " '5.971ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25390,8 +1490,8 @@ " '0 b',\n", " '0 b',\n", " '44'],\n", - " ['1.421ms',\n", - " '1.421ms',\n", + " ['1.455ms',\n", + " '1.455ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25399,8 +1499,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['539.000us',\n", - " '1.889ms',\n", + " ['626.000us',\n", + " '1.932ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25408,8 +1508,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.020ms',\n", - " '1.431ms',\n", + " ['1.094ms',\n", + " '1.507ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25417,8 +1517,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['480.000us',\n", - " '480.000us',\n", + " ['508.000us',\n", + " '508.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25426,53 +1526,53 @@ " '0 b',\n", " '0 b',\n", " '1413'],\n", - " ['3.339ms',\n", - " '156.911ms',\n", + " ['2.890ms',\n", + " '149.597ms',\n", " '0.000us',\n", - " '367.000us',\n", - " '0 b',\n", - " '0 b',\n", + " '334.000us',\n", " '0 b',\n", " '0 b',\n", + " '-10.30 Mb',\n", + " '697.92 Mb',\n", " '471'],\n", - " ['1.282ms',\n", - " '93.703ms',\n", + " ['2.133ms',\n", + " '89.208ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '10.30 Mb',\n", + " '697.92 Mb',\n", " '471'],\n", - " ['93.156ms',\n", - " '93.156ms',\n", + " ['88.735ms',\n", + " '88.735ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '697.92 Mb',\n", + " '697.92 Mb',\n", " '471'],\n", - " ['1.116ms',\n", - " '61.915ms',\n", + " ['1.005ms',\n", + " '59.082ms',\n", " '0.000us',\n", - " '381.000us',\n", + " '346.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['3.116ms',\n", - " '60.799ms',\n", - " '381.000us',\n", - " '381.000us',\n", + " ['2.999ms',\n", + " '58.077ms',\n", + " '346.000us',\n", + " '346.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['60.432ms',\n", - " '60.432ms',\n", + " ['57.526ms',\n", + " '57.526ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25482,17 +1582,17 @@ " '942'],\n", " ['0.000us',\n", " '0.000us',\n", - " '381.000us',\n", - " '381.000us',\n", + " '346.000us',\n", + " '346.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['7.711ms',\n", - " '10.529ms',\n", - " '1.307ms',\n", - " '1.307ms',\n", + " ['7.812ms',\n", + " '10.355ms',\n", + " '1.277ms',\n", + " '1.277ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25500,15 +1600,15 @@ " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.307ms',\n", - " '1.307ms',\n", + " '1.277ms',\n", + " '1.277ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.044ms',\n", - " '1.044ms',\n", + " ['1.048ms',\n", + " '1.048ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25516,17 +1616,17 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['10.282ms',\n", - " '164.389ms',\n", + " ['6.715ms',\n", + " '157.757ms',\n", " '0.000us',\n", - " '16.237ms',\n", - " '0 b',\n", - " '0 b',\n", + " '16.142ms',\n", " '0 b',\n", " '0 b',\n", + " '20.59 Mb',\n", + " '745.69 Mb',\n", " '471'],\n", - " ['2.049ms',\n", - " '3.278ms',\n", + " ['2.212ms',\n", + " '3.499ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25534,8 +1634,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['753.000us',\n", - " '1.229ms',\n", + " ['813.000us',\n", + " '1.287ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25543,8 +1643,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['476.000us',\n", - " '476.000us',\n", + " ['474.000us',\n", + " '474.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25552,23 +1652,23 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.070ms',\n", - " '159.753ms',\n", + " ['959.000us',\n", + " '152.888ms',\n", " '0.000us',\n", - " '17.112ms',\n", - " '0 b',\n", + " '16.628ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", + " '745.69 Mb',\n", " '471'],\n", - " ['95.770ms',\n", - " '158.683ms',\n", - " '17.112ms',\n", - " '17.112ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['92.151ms',\n", + " '151.929ms',\n", + " '16.628ms',\n", + " '16.628ms',\n", " '0 b',\n", " '0 b',\n", + " '745.69 Mb',\n", + " '745.69 Mb',\n", " '471'],\n", " ['1.000us',\n", " '1.000us',\n", @@ -25579,8 +1679,8 @@ " '0 b',\n", " '0 b',\n", " '1'],\n", - " ['152.000us',\n", - " '152.000us',\n", + " ['145.000us',\n", + " '145.000us',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25588,8 +1688,8 @@ " '0 b',\n", " '0 b',\n", " '1'],\n", - " ['1.023ms',\n", - " '1.023ms',\n", + " ['1.077ms',\n", + " '1.077ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25597,8 +1697,8 @@ " '0 b',\n", " '0 b',\n", " '511'],\n", - " ['32.548ms',\n", - " '32.548ms',\n", + " ['30.929ms',\n", + " '30.929ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25608,15 +1708,15 @@ " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '4.233ms',\n", - " '4.233ms',\n", + " '4.197ms',\n", + " '4.197ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '172'],\n", - " ['6.656ms',\n", - " '6.656ms',\n", + " ['6.041ms',\n", + " '6.041ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25624,8 +1724,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['29.189ms',\n", - " '29.189ms',\n", + " ['27.626ms',\n", + " '27.626ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25644,8 +1744,8 @@ " '230'],\n", " ['0.000us',\n", " '0.000us',\n", - " '9.892ms',\n", - " '9.892ms',\n", + " '9.588ms',\n", + " '9.588ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25653,8 +1753,8 @@ " '214'],\n", " ['0.000us',\n", " '0.000us',\n", - " '2.876ms',\n", - " '2.876ms',\n", + " '2.739ms',\n", + " '2.739ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25662,15 +1762,24 @@ " '83'],\n", " ['0.000us',\n", " '0.000us',\n", - " '111.000us',\n", - " '111.000us',\n", + " '104.000us',\n", + " '104.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '2'],\n", - " ['79.108ms',\n", - " '143.631ms',\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '-152.00 Kb',\n", + " '-152.00 Kb',\n", + " '744'],\n", + " ['70.478ms',\n", + " '126.577ms',\n", " '353.000us',\n", " '353.000us',\n", " '0 b',\n", @@ -25678,8 +1787,8 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['74.679ms',\n", - " '74.679ms',\n", + " ['65.908ms',\n", + " '65.908ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25696,64 +1805,64 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['1.191ms',\n", - " '84.025ms',\n", + " ['-532.000us',\n", + " '82.041ms',\n", " '0.000us',\n", - " '2.582ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.580ms',\n", " '0 b',\n", " '0 b',\n", + " '-3.91 Mb',\n", + " '149.22 Mb',\n", " '471'],\n", - " ['2.948ms',\n", - " '82.488ms',\n", + " ['4.488ms',\n", + " '80.580ms',\n", " '0.000us',\n", - " '2.566ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.540ms',\n", " '0 b',\n", " '0 b',\n", + " '3.60 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['30.383ms',\n", - " '30.383ms',\n", + " ['28.837ms',\n", + " '28.837ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.68 Mb',\n", + " '3.68 Mb',\n", " '1413'],\n", - " ['12.604ms',\n", - " '52.167ms',\n", - " '2.476ms',\n", - " '2.594ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['12.558ms',\n", + " '51.947ms',\n", + " '2.474ms',\n", + " '2.592ms',\n", " '0 b',\n", " '0 b',\n", + " '-5.31 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['750.000us',\n", - " '3.136ms',\n", + " ['722.000us',\n", + " '3.170ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '5.31 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.504ms',\n", - " '2.504ms',\n", + " ['2.523ms',\n", + " '2.523ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.064ms',\n", - " '1.064ms',\n", + " '1.061ms',\n", + " '1.061ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25770,15 +1879,15 @@ " '353'],\n", " ['0.000us',\n", " '0.000us',\n", - " '941.000us',\n", - " '941.000us',\n", + " '942.000us',\n", + " '942.000us',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.285ms',\n", - " '1.285ms',\n", + " ['1.191ms',\n", + " '1.191ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25786,8 +1895,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['747.000us',\n", - " '23.627ms',\n", + " ['781.000us',\n", + " '23.765ms',\n", " '118.000us',\n", " '118.000us',\n", " '0 b',\n", @@ -25795,8 +1904,8 @@ " '0 b',\n", " '0 b',\n", " '118'],\n", - " ['22.880ms',\n", - " '22.880ms',\n", + " ['22.984ms',\n", + " '22.984ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25822,8 +1931,17 @@ " '0 b',\n", " '0 b',\n", " '118'],\n", - " ['87.275ms',\n", - " '139.715ms',\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '-132.00 Kb',\n", + " '-132.00 Kb',\n", + " '739'],\n", + " ['81.224ms',\n", + " '129.447ms',\n", " '353.000us',\n", " '353.000us',\n", " '0 b',\n", @@ -25831,8 +1949,8 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['59.714ms',\n", - " '59.714ms',\n", + " ['55.220ms',\n", + " '55.220ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25849,64 +1967,64 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['528.000us',\n", - " '77.774ms',\n", + " ['661.000us',\n", + " '72.788ms',\n", " '0.000us',\n", - " '2.578ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.571ms',\n", " '0 b',\n", " '0 b',\n", + " '-2.89 Mb',\n", + " '149.20 Mb',\n", " '471'],\n", - " ['3.016ms',\n", - " '76.477ms',\n", + " ['2.810ms',\n", + " '71.489ms',\n", " '0.000us',\n", - " '2.548ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.533ms',\n", " '0 b',\n", " '0 b',\n", + " '2.98 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['32.189ms',\n", - " '32.189ms',\n", + " ['29.551ms',\n", + " '29.551ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.68 Mb',\n", + " '3.68 Mb',\n", " '1413'],\n", - " ['11.195ms',\n", - " '44.783ms',\n", - " '2.478ms',\n", - " '2.596ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['11.271ms',\n", + " '42.637ms',\n", + " '2.471ms',\n", + " '2.589ms',\n", " '0 b',\n", " '0 b',\n", + " '-3.47 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['634.000us',\n", - " '3.093ms',\n", + " ['667.000us',\n", + " '3.107ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.47 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.520ms',\n", - " '2.520ms',\n", + " ['2.492ms',\n", + " '2.492ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.065ms',\n", - " '1.065ms',\n", + " '1.058ms',\n", + " '1.058ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -25930,8 +2048,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.066ms',\n", - " '1.066ms',\n", + " ['1.099ms',\n", + " '1.099ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25939,8 +2057,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['745.000us',\n", - " '20.607ms',\n", + " ['776.000us',\n", + " '18.528ms',\n", " '118.000us',\n", " '118.000us',\n", " '0 b',\n", @@ -25948,8 +2066,8 @@ " '0 b',\n", " '0 b',\n", " '118'],\n", - " ['19.862ms',\n", - " '19.862ms',\n", + " ['17.752ms',\n", + " '17.752ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -25975,8 +2093,17 @@ " '0 b',\n", " '0 b',\n", " '118'],\n", - " ['86.501ms',\n", - " '132.128ms',\n", + " ['0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0.000us',\n", + " '0 b',\n", + " '0 b',\n", + " '-140.00 Kb',\n", + " '-140.00 Kb',\n", + " '741'],\n", + " ['79.923ms',\n", + " '123.217ms',\n", " '353.000us',\n", " '353.000us',\n", " '0 b',\n", @@ -25984,8 +2111,8 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['52.722ms',\n", - " '52.722ms',\n", + " ['50.052ms',\n", + " '50.052ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -26002,64 +2129,64 @@ " '0 b',\n", " '0 b',\n", " '353'],\n", - " ['873.000us',\n", - " '72.646ms',\n", + " ['1.028ms',\n", + " '69.357ms',\n", " '0.000us',\n", - " '2.573ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.562ms',\n", " '0 b',\n", " '0 b',\n", + " '-1.35 Mb',\n", + " '149.21 Mb',\n", " '471'],\n", - " ['2.720ms',\n", - " '71.364ms',\n", + " ['2.529ms',\n", + " '68.094ms',\n", " '0.000us',\n", - " '2.539ms',\n", - " '0 b',\n", - " '0 b',\n", + " '2.560ms',\n", " '0 b',\n", " '0 b',\n", + " '1.79 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['30.893ms',\n", - " '30.893ms',\n", + " ['28.910ms',\n", + " '28.910ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.68 Mb',\n", + " '3.68 Mb',\n", " '1413'],\n", - " ['10.998ms',\n", - " '40.990ms',\n", - " '2.479ms',\n", - " '2.597ms',\n", - " '0 b',\n", - " '0 b',\n", + " ['11.237ms',\n", + " '39.924ms',\n", + " '2.474ms',\n", + " '2.592ms',\n", " '0 b',\n", " '0 b',\n", + " '-3.13 Mb',\n", + " '150.00 Mb',\n", " '471'],\n", - " ['632.000us',\n", - " '3.084ms',\n", + " ['659.000us',\n", + " '3.080ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '3.13 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", - " ['2.510ms',\n", - " '2.510ms',\n", + " ['2.473ms',\n", + " '2.473ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", " '0 b',\n", - " '0 b',\n", - " '0 b',\n", + " '146.32 Mb',\n", + " '146.32 Mb',\n", " '471'],\n", " ['0.000us',\n", " '0.000us',\n", - " '1.066ms',\n", - " '1.066ms',\n", + " '1.061ms',\n", + " '1.061ms',\n", " '0 b',\n", " '0 b',\n", " '0 b',\n", @@ -26083,8 +2210,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['1.048ms',\n", - " '1.048ms',\n", + " ['1.042ms',\n", + " '1.042ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -26092,8 +2219,8 @@ " '0 b',\n", " '0 b',\n", " '471'],\n", - " ['745.000us',\n", - " '17.173ms',\n", + " ['777.000us',\n", + " '16.076ms',\n", " '118.000us',\n", " '118.000us',\n", " '0 b',\n", @@ -26101,8 +2228,8 @@ " '0 b',\n", " '0 b',\n", " '118'],\n", - " ['16.428ms',\n", - " '16.428ms',\n", + " ['15.299ms',\n", + " '15.299ms',\n", " '0.000us',\n", " '0.000us',\n", " '0 b',\n", @@ -26141,7 +2268,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -26151,7 +2278,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -26160,7 +2287,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -26169,7 +2296,7 @@ "10" ] }, - "execution_count": 19, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -26180,7 +2307,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -26198,7 +2325,7 @@ " 'Number of Calls']" ] }, - "execution_count": 20, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -26209,7 +2336,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -26218,7 +2345,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -26256,8 +2383,8 @@ " \n", " \n", " -dropout--aten::dropout\n", - " 136.000us\n", - " 136.000us\n", + " 142.000us\n", + " 142.000us\n", " 0.000us\n", " 0.000us\n", " 0 b\n", @@ -26268,8 +2395,8 @@ " \n", " \n", " -dropout--cudaDeviceSynchronize\n", - " 4.750ms\n", - " 4.750ms\n", + " 5.410ms\n", + " 5.410ms\n", " 0.000us\n", " 0.000us\n", " 0 b\n", @@ -26280,32 +2407,32 @@ " \n", " \n", " -act--aten::relu\n", - " 280.710ms\n", - " 520.573ms\n", + " 252.233ms\n", + " 476.470ms\n", " 0.000us\n", " 1.413ms\n", " 0 b\n", " 0 b\n", " 0 b\n", - " 0 b\n", + " 438.95 Mb\n", " 1413\n", " \n", " \n", " -act--aten::clamp_min\n", - " 22.064ms\n", - " 239.863ms\n", + " 23.247ms\n", + " 224.237ms\n", " 1.413ms\n", " 1.413ms\n", " 0 b\n", " 0 b\n", - " 0 b\n", - " 0 b\n", + " 438.95 Mb\n", + " 438.95 Mb\n", " 1413\n", " \n", " \n", " -act--cudaLaunchKernel\n", - " 217.799ms\n", - " 217.799ms\n", + " 200.990ms\n", + " 200.990ms\n", " 0.000us\n", " 0.000us\n", " 0 b\n", @@ -26328,8 +2455,8 @@ " \n", " \n", " -norms-- -cudaDeviceSynchronize\n", - " 1.048ms\n", - " 1.048ms\n", + " 1.042ms\n", + " 1.042ms\n", " 0.000us\n", " 0.000us\n", " 0 b\n", @@ -26340,8 +2467,8 @@ " \n", " \n", " -norms-- -aten::copy_\n", - " 745.000us\n", - " 17.173ms\n", + " 777.000us\n", + " 16.076ms\n", " 118.000us\n", " 118.000us\n", " 0 b\n", @@ -26352,8 +2479,8 @@ " \n", " \n", " -norms-- -cudaMemcpyAsync\n", - " 16.428ms\n", - " 16.428ms\n", + " 15.299ms\n", + " 15.299ms\n", " 0.000us\n", " 0.000us\n", " 0 b\n", @@ -26388,20 +2515,20 @@ " \n", " \n", "\n", - "

180 rows × 9 columns

\n", + "

183 rows × 9 columns

\n", "" ], "text/plain": [ " Self CPU total CPU total \\\n", - "-dropout--aten::dropout 136.000us 136.000us \n", - "-dropout--cudaDeviceSynchronize 4.750ms 4.750ms \n", - "-act--aten::relu 280.710ms 520.573ms \n", - "-act--aten::clamp_min 22.064ms 239.863ms \n", - "-act--cudaLaunchKernel 217.799ms 217.799ms \n", + "-dropout--aten::dropout 142.000us 142.000us \n", + "-dropout--cudaDeviceSynchronize 5.410ms 5.410ms \n", + "-act--aten::relu 252.233ms 476.470ms \n", + "-act--aten::clamp_min 23.247ms 224.237ms \n", + "-act--cudaLaunchKernel 200.990ms 200.990ms \n", "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 1.048ms 1.048ms \n", - "-norms-- -aten::copy_ 745.000us 17.173ms \n", - "-norms-- -cudaMemcpyAsync 16.428ms 16.428ms \n", + "-norms-- -cudaDeviceSynchronize 1.042ms 1.042ms \n", + "-norms-- -aten::copy_ 777.000us 16.076ms \n", + "-norms-- -cudaMemcpyAsync 15.299ms 15.299ms \n", "-norms-- -Memcpy DtoD (Device -> Device) 0.000us 0.000us \n", "-norms-- -void at::native::vectorized_elementwi... 0.000us 0.000us \n", "\n", @@ -26431,18 +2558,18 @@ "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", "\n", - " Self CUDA Mem CUDA Mem \\\n", - "-dropout--aten::dropout 0 b 0 b \n", - "-dropout--cudaDeviceSynchronize 0 b 0 b \n", - "-act--aten::relu 0 b 0 b \n", - "-act--aten::clamp_min 0 b 0 b \n", - "-act--cudaLaunchKernel 0 b 0 b \n", - "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", - "-norms-- -aten::copy_ 0 b 0 b \n", - "-norms-- -cudaMemcpyAsync 0 b 0 b \n", - "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", - "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", + " Self CUDA Mem CUDA Mem \\\n", + "-dropout--aten::dropout 0 b 0 b \n", + "-dropout--cudaDeviceSynchronize 0 b 0 b \n", + "-act--aten::relu 0 b 438.95 Mb \n", + "-act--aten::clamp_min 438.95 Mb 438.95 Mb \n", + "-act--cudaLaunchKernel 0 b 0 b \n", + "... ... ... \n", + "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", + "-norms-- -aten::copy_ 0 b 0 b \n", + "-norms-- -cudaMemcpyAsync 0 b 0 b \n", + "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", + "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", "\n", " Number of Calls \n", "-dropout--aten::dropout 1413 \n", @@ -26457,10 +2584,10 @@ "-norms-- -Memcpy DtoD (Device -> Device) 118 \n", "-norms-- -void at::native::vectorized_elementwi... 118 \n", "\n", - "[180 rows x 9 columns]" + "[183 rows x 9 columns]" ] }, - "execution_count": 22, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -26469,6 +2596,85 @@ "df" ] }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('llm_to_use', LLM(\n", + " (llm): LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32001, 2048)\n", + " (layers): ModuleList(\n", + " (0-21): 22 x LlamaDecoderLayer(\n", + " (self_attn): LlamaSdpaAttention(\n", + " (q_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", + " (k_proj): Linear(in_features=2048, out_features=256, bias=False)\n", + " (v_proj): Linear(in_features=2048, out_features=256, bias=False)\n", + " (o_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", + " (up_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", + " (down_proj): Linear(in_features=5632, out_features=2048, bias=False)\n", + " (act_fn): SiLU()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", + " )\n", + " (norm): LlamaRMSNorm()\n", + " )\n", + " (lm_head): Linear(in_features=2048, out_features=32001, bias=False)\n", + " )\n", + " (word_embedding): Embedding(32001, 2048)\n", + "))\n", + "('llm_generator', LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32001, 2048)\n", + " (layers): ModuleList(\n", + " (0-21): 22 x LlamaDecoderLayer(\n", + " (self_attn): LlamaSdpaAttention(\n", + " (q_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", + " (k_proj): Linear(in_features=2048, out_features=256, bias=False)\n", + " (v_proj): Linear(in_features=2048, out_features=256, bias=False)\n", + " (o_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", + " (up_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", + " (down_proj): Linear(in_features=5632, out_features=2048, bias=False)\n", + " (act_fn): SiLU()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", + " )\n", + " (norm): LlamaRMSNorm()\n", + " )\n", + " (lm_head): Linear(in_features=2048, out_features=32001, bias=False)\n", + "))\n", + "('graph_encoder', GAT(1024, 1024, num_layers=4))\n", + "('projector', Sequential(\n", + " (0): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (1): Sigmoid()\n", + " (2): Linear(in_features=1024, out_features=2048, bias=True)\n", + "))\n", + "('word_embedding', Embedding(32001, 2048))\n" + ] + } + ], + "source": [ + "for c in model.named_children():\n", + " print(c)" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/torch_geometric/profile/__init__.py b/torch_geometric/profile/__init__.py index 833ee657d0e7..22d3039f4c83 100644 --- a/torch_geometric/profile/__init__.py +++ b/torch_geometric/profile/__init__.py @@ -20,6 +20,7 @@ get_gpu_memory_from_nvidia_smi, get_model_size, ) +from .nvtx import nvtxit __all__ = [ 'profileit', @@ -38,6 +39,7 @@ 'get_gpu_memory_from_nvidia_smi', 'get_gpu_memory_from_ipex', 'benchmark', + 'nvtxit', ] classes = __all__ diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py new file mode 100644 index 000000000000..5c8be58bf757 --- /dev/null +++ b/torch_geometric/profile/nvtx.py @@ -0,0 +1,27 @@ +import torch +from typing import Optional +from functools import wraps + + +def nvtxit(name: str, n_warmups: int = 0, n_iters: Optional[int] = None): + def nvtx(func): + iters_so_far = 0 + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal iters_so_far + if not torch.cuda.is_available(): + return func(*args, **kwargs) + elif iters_so_far < n_warmups: + iters_so_far += 1 + return func(*args, **kwargs) + elif n_iters is None or iters_so_far < n_iters + n_warmups: + torch.cuda.nvtx.range_push(f"{name}_{iters_so_far}") + result = func(*args, **kwargs) + torch.cuda.nvtx.range_pop() + iters_so_far += 1 + return result + else: + return func(*args, **kwargs) + return wrapper + return nvtx From 5c3876dbebb23dcdde65ec82c651884a64a46fa6 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 27 Jun 2024 17:00:53 -0700 Subject: [PATCH 628/752] nvtx profiler optional name --- torch_geometric/profile/nvtx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py index 5c8be58bf757..7a291baff9e1 100644 --- a/torch_geometric/profile/nvtx.py +++ b/torch_geometric/profile/nvtx.py @@ -2,7 +2,6 @@ from typing import Optional from functools import wraps - def nvtxit(name: str, n_warmups: int = 0, n_iters: Optional[int] = None): def nvtx(func): iters_so_far = 0 From a2b606cc1232c16aa1cd60f05a0a348f253de128 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 27 Jun 2024 17:06:31 -0700 Subject: [PATCH 629/752] nvtx profiler optional name 2 --- torch_geometric/profile/nvtx.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py index 7a291baff9e1..9dbd41c0469b 100644 --- a/torch_geometric/profile/nvtx.py +++ b/torch_geometric/profile/nvtx.py @@ -2,9 +2,13 @@ from typing import Optional from functools import wraps -def nvtxit(name: str, n_warmups: int = 0, n_iters: Optional[int] = None): +def nvtxit(name: Optional[str] = None, n_warmups: int = 0, n_iters: Optional[int] = None): def nvtx(func): + + nonlocal name iters_so_far = 0 + if name is None: + name = func.__name__ @wraps(func) def wrapper(*args, **kwargs): From 0a588f6f05da5bb5825d704384476a30c48a5ee8 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 27 Jun 2024 17:12:13 -0700 Subject: [PATCH 630/752] precommit --- torch_geometric/profile/nvtx.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py index 9dbd41c0469b..26852d20d7d1 100644 --- a/torch_geometric/profile/nvtx.py +++ b/torch_geometric/profile/nvtx.py @@ -1,8 +1,11 @@ -import torch -from typing import Optional from functools import wraps +from typing import Optional + +import torch + -def nvtxit(name: Optional[str] = None, n_warmups: int = 0, n_iters: Optional[int] = None): +def nvtxit(name: Optional[str] = None, n_warmups: int = 0, + n_iters: Optional[int] = None): def nvtx(func): nonlocal name @@ -26,5 +29,7 @@ def wrapper(*args, **kwargs): return result else: return func(*args, **kwargs) + return wrapper + return nvtx From f0d7aa38a51f947832daf229af29982f4f1ef574 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 27 Jun 2024 17:45:20 -0700 Subject: [PATCH 631/752] nvtx shell script --- examples/llm_plus_gnn/test_nvtx_uwebqsp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/llm_plus_gnn/test_nvtx_uwebqsp.py diff --git a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py new file mode 100644 index 000000000000..3b83d36c9a7b --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py @@ -0,0 +1,18 @@ +from torch_geometric.datasets import updated_web_qsp_dataset +from torch_geometric.profile import nvtxit +import torch + +# Apply Patches +updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph = nvtxit(n_iters=1)(updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph) +updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs = nvtxit(n_iters=1)(updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs) +updated_web_qsp_dataset.retrieval_via_pcst = nvtxit(n_iters=10)(updated_web_qsp_dataset.retrieval_via_pcst) + +updated_web_qsp_dataset.get_features_for_triplets_groups = nvtxit()(updated_web_qsp_dataset.get_features_for_triplets_groups) +updated_web_qsp_dataset.LargeGraphIndexer.get_node_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features) +updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features) +updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter = nvtxit(n_warmups=1, n_iters=10)(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter) +updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter = nvtxit(n_warmups=1, n_iters=10)(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter) + +if __name__ == "__main__": + with torch.autograd.profiler.emit_nvtx(): + ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('small_ds', force_reload=True, limit=10) \ No newline at end of file From 8a4d0cf01f77f5466bf89162bd76ff05e83f72f2 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 27 Jun 2024 18:38:37 -0700 Subject: [PATCH 632/752] nvtx scripts added for kg loading and backend retrieval --- .../llm_plus_gnn/test_nvtx_rag_backend.py | 75 +++++++++++++++++++ examples/llm_plus_gnn/test_nvtx_webqsp.py | 11 +++ 2 files changed, 86 insertions(+) create mode 100644 examples/llm_plus_gnn/test_nvtx_rag_backend.py create mode 100644 examples/llm_plus_gnn/test_nvtx_webqsp.py diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/test_nvtx_rag_backend.py new file mode 100644 index 000000000000..190a0aece1f8 --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_rag_backend.py @@ -0,0 +1,75 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import rag_loader +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +from torch_geometric.profile.nvtx import nvtxit +import torch + +# %% +# Patch FeatureStore and GraphStore + +SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit(n_iter=10)(SentenceTransformerFeatureStore.retrieve_seed_nodes) +SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit(n_iters=10)(SentenceTransformerFeatureStore.retrieve_seed_edges) +SentenceTransformerFeatureStore.load_subgraph = nvtxit(n_iters=10)(SentenceTransformerFeatureStore.load_subgraph) +NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit(n_iters=10)(NeighborSamplingRAGGraphStore.sample_subgraph) + +# %% +ds = UpdatedWebQSPDataset("small_ds", force_reload=True, limit=10) + +# %% +triplets = list(chain.from_iterable((d['graph'] for d in ds.raw_dataset))) + +# %% +questions = ds.raw_dataset['question'] + +# %% +ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +num_edges = len(ds.indexer._edges) + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer().to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}) + +# %% +# Accuracy Metrics to be added to Profiler +def _eidx_helper(subg: Data, ground_truth: Data): + subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx + if isinstance(subg_eidx, torch.Tensor): + subg_eidx = subg_eidx.tolist() + if isinstance(gt_eidx, torch.Tensor): + gt_eidx = gt_eidx.tolist() + subg_e = set(subg_eidx) + gt_e = set(gt_eidx) + return subg_e, gt_e +def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + total_e = set(range(num_edges)) + tp = len(subg_e & gt_e) + tn = len(total_e-(subg_e | gt_e)) + return (tp+tn)/num_edges +def check_retrieval_precision(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(subg_e) +def check_retrieval_recall(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(gt_e) + + +# %% + +if __name__ == "__main__": + with torch.autograd.profiler.emit_nvtx(): + for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): + print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py new file mode 100644 index 000000000000..c67754b6bdc8 --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_webqsp.py @@ -0,0 +1,11 @@ +from torch_geometric.datasets import web_qsp_dataset +from torch_geometric.profile import nvtxit +import torch + +# Apply Patches +web_qsp_dataset.retrieval_via_pcst = nvtxit(n_warmups=1, n_iters=10)(web_qsp_dataset.retrieval_via_pcst) + + +if __name__ == "__main__": + with torch.autograd.profiler.emit_nvtx(): + ds = web_qsp_dataset.WebQSPDataset('baseline') \ No newline at end of file From 01696e4100d06a0056d81f7209268308f5e241c9 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 28 Jun 2024 14:30:07 -0700 Subject: [PATCH 633/752] nvtx_webqsp_script --- examples/llm_plus_gnn/test_nvtx_webqsp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py index c67754b6bdc8..c35b9ff18821 100644 --- a/examples/llm_plus_gnn/test_nvtx_webqsp.py +++ b/examples/llm_plus_gnn/test_nvtx_webqsp.py @@ -4,6 +4,7 @@ # Apply Patches web_qsp_dataset.retrieval_via_pcst = nvtxit(n_warmups=1, n_iters=10)(web_qsp_dataset.retrieval_via_pcst) +web_qsp_dataset.WebQSPDataset.process = nvtxit()(web_qsp_dataset.WebQSPDataset.process) if __name__ == "__main__": From 42cc16ec519cd2f6287ad6dbc268227cfa4cf680 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 28 Jun 2024 16:00:29 -0700 Subject: [PATCH 634/752] cudaprofilerapi hook --- examples/llm_plus_gnn/nvtx_run.sh | 27 +++++++++++++++++++++++++++ torch_geometric/profile/nvtx.py | 20 ++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 examples/llm_plus_gnn/nvtx_run.sh diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_run.sh new file mode 100755 index 000000000000..7e08ee519468 --- /dev/null +++ b/examples/llm_plus_gnn/nvtx_run.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Check if the user provided a Python file +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Check if the provided file exists +if [[ ! -f "$1" ]]; then + echo "Error: File '$1' does not exist." + exit 1 +fi + +# Check if the provided file is a Python file +if [[ ! "$1" == *.py ]]; then + echo "Error: '$1' is not a Python file." + exit 1 +fi + +# Get the base name of the Python file +python_file=$(basename "$1") + +# Run nsys profile on the Python file +nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cudabacktrace true --force-overwrite true --output=profile_${python_file%.py} python "$1" + +echo "Profile data saved as profile_${python_file%.py}.nsys-rep" \ No newline at end of file diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py index 26852d20d7d1..122ddfad916f 100644 --- a/torch_geometric/profile/nvtx.py +++ b/torch_geometric/profile/nvtx.py @@ -3,6 +3,24 @@ import torch +CUDA_PROFILE_STARTED = False + + +def begin_cuda_profile(): + global CUDA_PROFILE_STARTED + prev_state = CUDA_PROFILE_STARTED + if prev_state is False: + CUDA_PROFILE_STARTED = True + torch.cuda.cudart().cudaProfilerStart() + return prev_state + + +def end_cuda_profile(prev_state: bool): + global CUDA_PROFILE_STARTED + CUDA_PROFILE_STARTED = prev_state + if prev_state is False: + torch.cuda.cudart().cudaProfilerStop() + def nvtxit(name: Optional[str] = None, n_warmups: int = 0, n_iters: Optional[int] = None): @@ -22,9 +40,11 @@ def wrapper(*args, **kwargs): iters_so_far += 1 return func(*args, **kwargs) elif n_iters is None or iters_so_far < n_iters + n_warmups: + prev_state = begin_cuda_profile() torch.cuda.nvtx.range_push(f"{name}_{iters_so_far}") result = func(*args, **kwargs) torch.cuda.nvtx.range_pop() + end_cuda_profile(prev_state) iters_so_far += 1 return result else: From d390cce7679de24bf41ddfd0fc015d4d076e6188 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 28 Jun 2024 19:19:50 -0700 Subject: [PATCH 635/752] make tracking the kernels optional --- examples/llm_plus_gnn/nvtx_run.sh | 2 +- .../llm_plus_gnn/test_nvtx_rag_backend.py | 11 +++++++-- examples/llm_plus_gnn/test_nvtx_uwebqsp.py | 23 +++++++++++++------ examples/llm_plus_gnn/test_nvtx_webqsp.py | 8 ++++++- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_run.sh index 7e08ee519468..b40c32abb0ea 100755 --- a/examples/llm_plus_gnn/nvtx_run.sh +++ b/examples/llm_plus_gnn/nvtx_run.sh @@ -22,6 +22,6 @@ fi python_file=$(basename "$1") # Run nsys profile on the Python file -nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cudabacktrace true --force-overwrite true --output=profile_${python_file%.py} python "$1" +nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cuda-memory-usage true --cudabacktrace all --force-overwrite true --output=profile_${python_file%.py} python "$1" echo "Profile data saved as profile_${python_file%.py}.nsys-rep" \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/test_nvtx_rag_backend.py index 190a0aece1f8..60df54f5c5f3 100644 --- a/examples/llm_plus_gnn/test_nvtx_rag_backend.py +++ b/examples/llm_plus_gnn/test_nvtx_rag_backend.py @@ -70,6 +70,13 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): # %% if __name__ == "__main__": - with torch.autograd.profiler.emit_nvtx(): + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): + print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + else: for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): - print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py index 3b83d36c9a7b..6204fe603b84 100644 --- a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py +++ b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py @@ -1,18 +1,27 @@ from torch_geometric.datasets import updated_web_qsp_dataset from torch_geometric.profile import nvtxit import torch +import argparse # Apply Patches -updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph = nvtxit(n_iters=1)(updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph) -updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs = nvtxit(n_iters=1)(updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs) -updated_web_qsp_dataset.retrieval_via_pcst = nvtxit(n_iters=10)(updated_web_qsp_dataset.retrieval_via_pcst) +updated_web_qsp_dataset.UpdatedWebQSPDataset.process = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset.process) +updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph) +updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs) +updated_web_qsp_dataset.SentenceTransformer.encode = nvtxit()(updated_web_qsp_dataset.SentenceTransformer.encode) +updated_web_qsp_dataset.retrieval_via_pcst = nvtxit()(updated_web_qsp_dataset.retrieval_via_pcst) updated_web_qsp_dataset.get_features_for_triplets_groups = nvtxit()(updated_web_qsp_dataset.get_features_for_triplets_groups) updated_web_qsp_dataset.LargeGraphIndexer.get_node_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features) updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features) -updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter = nvtxit(n_warmups=1, n_iters=10)(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter) -updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter = nvtxit(n_warmups=1, n_iters=10)(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter) +updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter) +updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter) if __name__ == "__main__": - with torch.autograd.profiler.emit_nvtx(): - ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('small_ds', force_reload=True, limit=10) \ No newline at end of file + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) + else: + ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py index c35b9ff18821..4859687e56d3 100644 --- a/examples/llm_plus_gnn/test_nvtx_webqsp.py +++ b/examples/llm_plus_gnn/test_nvtx_webqsp.py @@ -8,5 +8,11 @@ if __name__ == "__main__": - with torch.autograd.profiler.emit_nvtx(): + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + ds = web_qsp_dataset.WebQSPDataset('baseline') + else: ds = web_qsp_dataset.WebQSPDataset('baseline') \ No newline at end of file From 3c2a20da62e1be98841d94efca1c1f59f2da82aa Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 2 Jul 2024 10:37:55 -0700 Subject: [PATCH 636/752] nvtx test scripts --- .../llm_plus_gnn/test_nvtx_rag_backend.py | 39 ++++++++++++++----- examples/llm_plus_gnn/test_nvtx_webqsp.py | 3 +- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/test_nvtx_rag_backend.py index 60df54f5c5f3..b0f89bfcca61 100644 --- a/examples/llm_plus_gnn/test_nvtx_rag_backend.py +++ b/examples/llm_plus_gnn/test_nvtx_rag_backend.py @@ -10,17 +10,20 @@ from itertools import chain from torch_geometric.profile.nvtx import nvtxit import torch +import argparse +from typing import Tuple # %% # Patch FeatureStore and GraphStore -SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit(n_iter=10)(SentenceTransformerFeatureStore.retrieve_seed_nodes) -SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit(n_iters=10)(SentenceTransformerFeatureStore.retrieve_seed_edges) -SentenceTransformerFeatureStore.load_subgraph = nvtxit(n_iters=10)(SentenceTransformerFeatureStore.load_subgraph) -NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit(n_iters=10)(NeighborSamplingRAGGraphStore.sample_subgraph) +SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_nodes) +SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_edges) +SentenceTransformerFeatureStore.load_subgraph = nvtxit()(SentenceTransformerFeatureStore.load_subgraph) +NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()(NeighborSamplingRAGGraphStore.sample_subgraph) +rag_loader.RAGQueryLoader.query = nvtxit()(rag_loader.RAGQueryLoader.query) # %% -ds = UpdatedWebQSPDataset("small_ds", force_reload=True, limit=10) +ds = UpdatedWebQSPDataset("small_ds_1", force_reload=True, limit=10) # %% triplets = list(chain.from_iterable((d['graph'] for d in ds.raw_dataset))) @@ -40,7 +43,19 @@ fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() # %% -query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}) +from torch_geometric.datasets.updated_web_qsp_dataset import retrieval_via_pcst + +@nvtxit() +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + q_emb = model.encode(query) + textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph["desc"] = desc + return graph + +# %% +query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}, local_filter=apply_retrieval_via_pcst) # %% # Accuracy Metrics to be added to Profiler @@ -69,14 +84,18 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): # %% +@nvtxit() +def _run_eval(): + for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): + print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--capture-torch-kernels", "-k", action="store_true") args = parser.parse_args() if args.capture_torch_kernels: with torch.autograd.profiler.emit_nvtx(): - for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): - print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + _run_eval() else: - for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): - print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) \ No newline at end of file + _run_eval() \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py index 4859687e56d3..1db8e5934728 100644 --- a/examples/llm_plus_gnn/test_nvtx_webqsp.py +++ b/examples/llm_plus_gnn/test_nvtx_webqsp.py @@ -1,9 +1,10 @@ from torch_geometric.datasets import web_qsp_dataset from torch_geometric.profile import nvtxit import torch +import argparse # Apply Patches -web_qsp_dataset.retrieval_via_pcst = nvtxit(n_warmups=1, n_iters=10)(web_qsp_dataset.retrieval_via_pcst) +web_qsp_dataset.retrieval_via_pcst = nvtxit()(web_qsp_dataset.retrieval_via_pcst) web_qsp_dataset.WebQSPDataset.process = nvtxit()(web_qsp_dataset.WebQSPDataset.process) From e91a951f1ee6a926804fc4360fc66d7ed3ead74f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 3 Jul 2024 16:40:43 -0700 Subject: [PATCH 637/752] bugfixes --- torch_geometric/datasets/updated_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index c489151407a4..e7438d74f703 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -308,7 +308,7 @@ def _retrieve_subgraphs(self) -> None: def process(self) -> None: self._load_raw_data() - self.model = SentenceTransformer().to(self.device) + self.model = SentenceTransformer('sentence-transformers/all-roberta-large-v1').to(self.device) self.model.eval() if not os.path.exists(self.processed_paths[-1]): print("Encoding graph...") From 7ddee4d8fcd294a49ef9198c537032bd41d88d17 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 8 Jul 2024 16:09:58 -0700 Subject: [PATCH 638/752] code to parallelize checking step in g_retriever --- examples/llm_plus_gnn/g_retriever.py | 84 +++++++++++++++------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 9d649e4f4d8f..868406a57dc9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -23,13 +23,10 @@ from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM +import multiprocessing as mp - -def detect_hallucinate(pred: str, label: str): - r"""An approximation for the unsolved task of detecting hallucinations. - We define a hallucination as an output that contains no instances of - acceptable label. - """ +def _detect_hallucinate(inp): + pred, label = inp try: split_pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(split_pred[0], label)) > 0 @@ -41,6 +38,15 @@ def detect_hallucinate(pred: str, label: str): return "skip" +def detect_hallucinate(pred_batch, label_batch): + r"""An approximation for the unsolved task of detecting hallucinations. + We define a hallucination as an output that contains no instances of + acceptable label. + """ + with mp.Pool(len(pred_batch)) as p: + res = p.map(_detect_hallucinate, zip(pred_batch, label_batch)) + return res + def compute_accuracy(eval_output) -> float: df = pd.concat([pd.DataFrame(d) for d in eval_output]) all_hit = [] @@ -279,7 +285,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=1, drop_last=False, + loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, pin_memory=True, shuffle=False) if tiny_llama: pure_llm = LLM( @@ -312,8 +318,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa ) for i, batch in enumerate(tqdm(loader)): - question = batch.question[0] - correct_answer = batch.label[0] + question = batch.question + correct_answer = batch.label if skip_pretrained_LLM: pure_llm_pred = None pure_llm_hallucinates = False @@ -324,18 +330,19 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] + untuned_llm_save_list += [tup for tup in zip(pure_llm_pred, pure_llm_hallucinates)] - gnn_llm_pred = gnn_llm_preds[i] + gnn_llm_pred = gnn_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [(gnn_llm_pred, gnn_llm_hallucinates)] - - if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa - # skipping when hallucination is hard to eval - continue - gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) - pure_llm_hallucin_sum += int(pure_llm_hallucinates) + gnn_save_list += [tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates)] + + for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, pure_llm_hallucinates): + if gnn_llm_hal == "skip" or pure_llm_hal == "skip": # noqa + # skipping when hallucination is hard to eval + continue + gnn_llm_hallucin_sum += int(gnn_llm_hal) + pure_llm_hallucin_sum += int(pure_llm_hal) if not skip_pretrained_LLM: print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) @@ -386,30 +393,31 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_preds += out['pred'] print("Final comparison between all models...") for i, batch in enumerate(tqdm(loader)): - question = batch.question[0] - correct_answer = batch.label[0] - gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] - untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] + question = batch.question + correct_answer = batch.label + gnn_llm_pred, gnn_llm_hallucinates = list(zip(*gnn_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) + untuned_llm_pred, untuned_llm_hallucinates = list(zip(*untuned_llm_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm_preds[i] + pure_llm_pred = pure_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) - if pure_llm_hallucinates == "skip": - continue - trained_llm_hallucin_sum += int(pure_llm_hallucinates) - if skip_pretrained_LLM: - # we did not check the untrained LLM, so do not decide to demo - # based on this. - untuned_llm_hallucinates = True - if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa - final_prnt_str += "Prompt: '" + question + "'\n" - final_prnt_str += "Label: '" + correct_answer + "'\n" - if not skip_pretrained_LLM: - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa - final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" - final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" - final_prnt_str += "\n" + "#" * 20 + "\n\n" + for j in range(len(gnn_llm_pred)): + if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[j] == "skip" or pure_llm_hallucinates[j] == "skip": + continue + trained_llm_hallucin_sum += int(pure_llm_hallucinates[j]) + if skip_pretrained_LLM: + # we did not check the untrained LLM, so do not decide to demo + # based on this. + untuned_llm_hallucinates = True + if untuned_llm_hallucinates[j] and pure_llm_hallucinates[j] and not gnn_llm_hallucinates[j]: # noqa + final_prnt_str += "Prompt: '" + question[j] + "'\n" + final_prnt_str += "Label: '" + correct_answer[j] + "'\n" + if not skip_pretrained_LLM: + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[j] + "'\n" # noqa + final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[j] + "'\n" + final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred[j] + "'\n" + final_prnt_str += "\n" + "#" * 20 + "\n\n" if not skip_pretrained_LLM: print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) From 18eabf4becc7f475b4dc773e794f58e472fbc35a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 10 Jul 2024 19:08:22 -0700 Subject: [PATCH 639/752] refactor training and demo loop to allow for multiple model benchmarks --- examples/llm_plus_gnn/g_retriever.py | 142 +++++++- examples/llm_plus_gnn/test_g_retriever.ipynb | 363 +++++++++++++++++++ 2 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 examples/llm_plus_gnn/test_g_retriever.ipynb diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 868406a57dc9..0dfd4968dadd 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -16,6 +16,8 @@ import torch from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm +import torch.nn as nn +from typing import Callable, List, Type, Dict, Any from torch_geometric import seed_everything from torch_geometric.data import Dataset @@ -127,9 +129,10 @@ def inference_step(model, batch, model_save_name): batch.desc) + def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=None, dataset=None, - checkpointing=False, tiny_llama=False): + checkpointing=False, tiny_llama=False, model_save_name=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -156,6 +159,7 @@ def adjust_learning_rate(param_group, LR, epoch): train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] + print(len(train_dataset), len(val_dataset), len(test_dataset)) train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, pin_memory=True, shuffle=True) @@ -179,10 +183,11 @@ def adjust_learning_rate(param_group, LR, epoch): model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) - if num_gnn_layers is not None: - model_save_name = "gnn_llm" - else: - model_save_name = "llm" + if model_save_name is None: + if num_gnn_layers is not None: + model_save_name = "gnn_llm" + else: + model_save_name = "llm" # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -275,6 +280,133 @@ def adjust_learning_rate(param_group, LR, epoch): print("Done!") return prep_time, dataset, eval_output +def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): + model_save_list = [] + model_preds = [] + for out in outs: + model_preds += out['pred'] + for i, batch in enumerate(loader): + correct_answer = batch.label + + model_pred = model_preds[i*eval_batch_size:(i+1)*eval_batch_size] + model_hallucinates = detect_hallucinate(model_pred, + correct_answer) + model_save_list += [tup for tup in zip(model_pred, model_hallucinates)] + return model_save_list + + +def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True): + + model_log: Dict[str, Dict[str, Any]] = dict() + idx_split = dataset.split_idxs + test_dataset = [dataset[i] for i in idx_split['test']] + loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, + pin_memory=True, shuffle=False) + + if not skip_LLMs: + if tiny_llama: + pure_llm = LLM( + model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_params=1, + ) + else: + pure_llm = LLM(model_name="llama2-7b", num_params=7) + + if not path.exists("pure_llm_model_log.pt"): + model_log["pure_llm"] = dict() + + pure_preds = [] + for batch in tqdm(loader): + pure_llm_preds = pure_llm.inference(batch.question, batch.desc, max_tokens=256) + pure_preds += pure_llm_preds + pure_preds = [{"pred": pred} for pred in pure_preds] + + model_log["pure_llm"]["preds"] = pure_preds + model_log["pure_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(pure_preds, loader, eval_batch_size) + torch.save(model_log["pure_llm"], "pure_llm_model_log.pt") + else: + model_log["pure_llm"] = torch.load("pure_llm_model_log.pt") + + # LORA + if not path.exists("tuned_llm_model_log.pt"): + model_log["tuned_llm"] = dict() + since = time.time() + gc.collect() + prep_time, _, lora_eval_outs = train(since, epochs, None, None, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=pure_llm, dataset=dataset) + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() + e2e_time = round(time.time() - since, 2) + model_log["tuned_llm"]["prep_time"] = prep_time + model_log["tuned_llm"]["e2e_time"] = e2e_time + model_log["tuned_llm"]["eval_output"] = lora_eval_outs + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") + + model_log["tuned_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(lora_eval_outs, loader, eval_batch_size) + torch.save(model_log["tuned_llm"], "tuned_llm_model_log.pt") + else: + model_log["tuned_llm"] = torch.load("tuned_llm_model_log.pt") + + del pure_llm + gc.collect() + + # All other models + for name, Model, kwargs in zip(model_names, models, model_kwargs): + model_log[name] = dict() + train_model = True + if path.exists(f"{name}.pt"): + print(f"Model {name} appears to already exist.") + print("Would you like to retrain?") + train_model = str(input("(y/n):")).lower() == "y" + + if train_model: + since = time.time() + gc.collect() + model = Model(**kwargs) + prep_time, _, model_eval_outs = train( + since=since, num_epochs=epochs, hidden_channels=None, num_gnn_layers=None, + batch_size=batch_size, eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, + inference_fn=inference_fn, checkpointing=checkpointing, + tiny_llama=tiny_llama, dataset=dataset, model_save_name=name, model=model) + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() + e2e_time = round(time.time() - since, 2) + model_log[name]["prep_time"] = prep_time + model_log[name]["e2e_time"] = e2e_time + model_log[name]["eval_output"] = model_eval_outs + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") + del model + gc.collect() + else: + model_eval_outs = torch.load(f"{name}_eval_outs.pt") + + # Calculate Hallucinations + skip_hallucination_detection = False + + if path.exists(f"{name}_model_log.pt"): + print(f"Saved outputs for {name} have been found.") + print("Would you like to redo?") + user_input = str(input("(y/n):")).lower() + skip_hallucination_detection = user_input != "y" + + + if not skip_hallucination_detection: + model_save_list = _eval_hallucinations_on_loader(model_eval_outs, loader, eval_batch_size) + + model_log[name]["hallucinates_list"] = model_save_list + torch.save(model_log[name], f"{name}_model_log.pt") + else: + model_log[name]["hallucinates_list"] = torch.load(f"{name}_model_log.pt")["hallucinates_list"] + + hal_dict = {k: [tup[1] for tup in v["hallucinates_list"]] for (k, v) in model_log.items()} + hallucinates_df = pd.DataFrame(hal_dict) + for col in hallucinates_df.columns: + print(f"Hallucinates for {col}:") + print(hallucinates_df[col].value_counts()) + def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size, loss_fn, inference_fn, diff --git a/examples/llm_plus_gnn/test_g_retriever.ipynb b/examples/llm_plus_gnn/test_g_retriever.ipynb new file mode 100644 index 000000000000..fb507b0332ed --- /dev/null +++ b/examples/llm_plus_gnn/test_g_retriever.ipynb @@ -0,0 +1,363 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from g_retriever import benchmark_models, get_loss, inference_step\n", + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from torch_geometric.nn.models import GRetriever" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "hidden_channels = 1024\n", + "num_gnn_layers = 1\n", + "lr=1e-5\n", + "epochs=2\n", + "batch_size=8\n", + "eval_batch_size=16" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "ds = UpdatedWebQSPDataset('test_script', limit=32)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "model_names = [\"GAT_1\", \"MLP_1\"]\n", + "model_classes = [GRetriever, GRetriever]\n", + "model_kwargs = [dict(llm_to_use=\"TinyLlama/TinyLlama-1.1B-Chat-v0.1\", num_llm_params=1, gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers, mlp_out_dim=2048)] * 2" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torch_geometric/nn/nlp/llm.py:102: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n", + " self.autocast_context = torch.cuda.amp.autocast(dtype=dtype)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model GAT_1 appears to already exist.\n", + "Would you like to retrain?\n", + "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n", + "16 8 8\n", + "Total Prep Time (prep_time) = 1.87\n", + "Training beginning...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch: 1|2: 100%|██████████| 2/2 [00:00<00:00, 2.27it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 1|2,Train Loss (Epoch Mean): 3.2131307125091553\n", + "Epoch: 1|2, Val Loss: 2.994086265563965\n", + "Checkpointing best val loss model...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch: 2|2: 100%|██████████| 2/2 [00:00<00:00, 2.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 2|2,Train Loss (Epoch Mean): 2.8135846853256226\n", + "Epoch: 2|2, Val Loss: 2.8492681980133057\n", + "Checkpointing best val loss model...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torch/cuda/memory.py:350: FutureWarning: torch.cuda.reset_max_memory_allocated now calls torch.cuda.reset_peak_memory_stats, which resets /all/ peak memory stats.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final Evaluation...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1/1 [00:00<00:00, 1.47it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Label: dilma rousseff\n", + "Pred: ['[/SRC] [/EDGE] [/SRC] [/EDGE] [/SRC] [/EDGE] [/']\n", + "Exception: unterminated character set at position 45\n", + "------------------\n", + "Label: paris\n", + "Pred: ['[/QU] [/QC] [/Q] [/QA] [/QB] [/QC] [/QD']\n", + "Exception: unterminated character set at position 35\n", + "------------------\n", + "Label: ice hockey\n", + "Pred: ['[/SRC] [/EDGE] [/SRC,EDGE] [/EDGE,SRC] [/EDGE,ED']\n", + "Exception: unterminated character set at position 39\n", + "------------------\n", + "Label: chaz bono|elijah blue allman\n", + "Pred: ['A: You can use the following code:\\nimport pprint\\n\\ndef load_data():\\n with open(\"data.txt']\n", + "Exception: missing ), unterminated subpattern at position 80 (line 5, column 14)\n", + "------------------\n", + "Hit: 0.0000\n", + "Precision: 0.0000\n", + "Recall: 0.0000\n", + "F1: 0.0000\n", + "Test Acc 0.0\n", + "Saving Model...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1/1 [00:04<00:00, 4.01s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving eval output for downstream demo...\n", + "Done!\n", + "E2E time (e2e_time) = 14.79 seconds\n", + "E2E tme minus Prep Time = 12.919999999999998 seconds\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved outputs for GAT_1 have been found.\n", + "Would you like to redo?\n", + "Model MLP_1 appears to already exist.\n", + "Would you like to retrain?\n", + "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n", + "16 8 8\n", + "Total Prep Time (prep_time) = 1.76\n", + "Training beginning...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch: 1|2: 100%|██████████| 2/2 [00:00<00:00, 2.79it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 1|2,Train Loss (Epoch Mean): 3.2048009634017944\n", + "Epoch: 1|2, Val Loss: 3.015418291091919\n", + "Checkpointing best val loss model...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch: 2|2: 100%|██████████| 2/2 [00:00<00:00, 2.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 2|2,Train Loss (Epoch Mean): 2.8109872341156006\n", + "Epoch: 2|2, Val Loss: 2.8776462078094482\n", + "Checkpointing best val loss model...\n", + "Final Evaluation...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1/1 [00:00<00:00, 1.55it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Label: dilma rousseff\n", + "Pred: ['[/SRC] [/EDGE] [/SRC] [/EDGE] [/SRC] [/EDGE] [/']\n", + "Exception: unterminated character set at position 45\n", + "------------------\n", + "Label: paris\n", + "Pred: ['[/QUERY] [/S] [/C] [/A] [/P] [/F] [/R] [/L']\n", + "Exception: unterminated character set at position 39\n", + "------------------\n", + "Label: ice hockey\n", + "Pred: ['[/SRC] [/EDR] [/Q] [/P] [/T] [/E] [/L] [/']\n", + "Exception: unterminated character set at position 39\n", + "------------------\n", + "Hit: 0.0000\n", + "Precision: 0.0000\n", + "Recall: 0.0000\n", + "F1: 0.0000\n", + "Test Acc 0.0\n", + "Saving Model...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1/1 [00:04<00:00, 4.05s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving eval output for downstream demo...\n", + "Done!\n", + "E2E time (e2e_time) = 15.26 seconds\n", + "E2E tme minus Prep Time = 13.5 seconds\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved outputs for MLP_1 have been found.\n", + "Would you like to redo?\n", + "Hallucinates for pure_llm:\n", + "pure_llm\n", + "True 5\n", + "skip 2\n", + "False 1\n", + "Name: count, dtype: int64\n", + "Hallucinates for tuned_llm:\n", + "tuned_llm\n", + "True 6\n", + "skip 1\n", + "False 1\n", + "Name: count, dtype: int64\n", + "Hallucinates for GAT_1:\n", + "GAT_1\n", + "skip 4\n", + "True 2\n", + "False 2\n", + "Name: count, dtype: int64\n", + "Hallucinates for MLP_1:\n", + "MLP_1\n", + "skip 3\n", + "True 3\n", + "False 2\n", + "Name: count, dtype: int64\n" + ] + } + ], + "source": [ + "benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From e120760e508dcbcff70ca65a890ef066692b1e5f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 11 Jul 2024 14:30:49 -0700 Subject: [PATCH 640/752] benchmark script for current experiment --- .../llm_plus_gnn/benchmark_model_archs.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/llm_plus_gnn/benchmark_model_archs.py diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py new file mode 100644 index 000000000000..e4ba2af46923 --- /dev/null +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -0,0 +1,37 @@ +# %% +from .g_retriever import benchmark_models, get_loss, inference_step +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.models import GRetriever, MLP, GAT + +# %% +hidden_channels = 1024 +num_gnn_layers = 1 +lr=1e-5 +epochs=2 +batch_size=8 +eval_batch_size=16 + +# %% +ds = UpdatedWebQSPDataset('benchmark_archs') + +# %% +model_names = [] +model_classes = [] +model_kwargs = [] +model_type = ["GAT", "MLP"] +models = {"GAT": GAT, "MLP": MLP} +num_layers = [1, 2, 4, 8, 16, 32] +for m_type in model_type: + for n_layer in num_layers: + model_names.append(f"{m_type}_{n_layer}") + model_classes.append(GRetriever) + kwargs = dict(gnn_hidden_channels=hidden_channels, num_gnn_layers=n_layer, gnn_to_use=models[m_type]) + model_kwargs.append(kwargs) + +# %% +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) + +# %% + + + From 71788deaae50e0e8ddf37bd6adb430c1956f1e76 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 15 Jul 2024 17:40:24 -0700 Subject: [PATCH 641/752] gretriever more embedding size --- .../llm_plus_gnn/benchmark_model_archs.py | 2 +- .../llm_plus_gnn/benchmark_model_gnn_enc.py | 39 +++++++++++++++++ examples/llm_plus_gnn/g_retriever.py | 42 +++++++++++-------- torch_geometric/nn/models/g_retriever.py | 5 ++- 4 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 examples/llm_plus_gnn/benchmark_model_gnn_enc.py diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index e4ba2af46923..f73d16b7ed9d 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -1,5 +1,5 @@ # %% -from .g_retriever import benchmark_models, get_loss, inference_step +from g_retriever import benchmark_models, get_loss, inference_step from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.models import GRetriever, MLP, GAT diff --git a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py new file mode 100644 index 000000000000..db1210b70ed4 --- /dev/null +++ b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py @@ -0,0 +1,39 @@ +# %% +from g_retriever import benchmark_models, get_loss, inference_step +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.models import GRetriever, MLP, GAT + +# %% +hidden_channels = 1024 +num_gnn_layers = 1 +lr=1e-5 +epochs=2 +batch_size=8 +eval_batch_size=16 + +# %% +ds = UpdatedWebQSPDataset('benchmark_archs') + +# %% +model_names = [] +model_classes = [] +model_kwargs = [] +model_type = ["GAT"] +models = {"GAT": GAT, "MLP": MLP} +num_layers = [4] +num_tokens = [4, 16] +for m_type in model_type: + for n_layer in num_layers: + for n_tokens in num_tokens + model_names.append(f"{m_type}_{n_layer}_{n_tokens}") + model_classes.append(GRetriever) + kwargs = dict(gnn_hidden_channels=hidden_channels*n_tokens, num_gnn_layers=n_layer, gnn_to_use=models[m_type], gnn_out_channels=hidden_channels*n_tokens, mlp_out_tokens=n_tokens) + model_kwargs.append(kwargs) + +# %% +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) + +# %% + + + diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0dfd4968dadd..1d0956ed4840 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -92,6 +92,9 @@ def compute_accuracy(eval_output) -> float: return hit +def compute_n_parameters(model: torch.nn.Module) -> int: + return sum([p.numel() for p in model.parameters() if p.requires_grad]) + def save_params_dict(model, save_path): state_dict = model.state_dict() @@ -295,7 +298,7 @@ def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): return model_save_list -def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True): +def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True, force: bool = False, root_dir='.'): model_log: Dict[str, Dict[str, Any]] = dict() idx_split = dataset.split_idxs @@ -312,7 +315,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode else: pure_llm = LLM(model_name="llama2-7b", num_params=7) - if not path.exists("pure_llm_model_log.pt"): + if not path.exists(root_dir +"/pure_llm_model_log.pt"): model_log["pure_llm"] = dict() pure_preds = [] @@ -323,12 +326,13 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode model_log["pure_llm"]["preds"] = pure_preds model_log["pure_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(pure_preds, loader, eval_batch_size) - torch.save(model_log["pure_llm"], "pure_llm_model_log.pt") + model_log["pure_llm"]["n_params"] = compute_n_parameters(pure_llm) + torch.save(model_log["pure_llm"], root_dir+"/pure_llm_model_log.pt") else: - model_log["pure_llm"] = torch.load("pure_llm_model_log.pt") + model_log["pure_llm"] = torch.load(root_dir+"/pure_llm_model_log.pt") # LORA - if not path.exists("tuned_llm_model_log.pt"): + if not path.exists(root_dir+"/tuned_llm_model_log.pt"): model_log["tuned_llm"] = dict() since = time.time() gc.collect() @@ -344,9 +348,10 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") model_log["tuned_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(lora_eval_outs, loader, eval_batch_size) - torch.save(model_log["tuned_llm"], "tuned_llm_model_log.pt") + model_log["tuned_llm"]["n_params"] = compute_n_parameters(pure_llm) + torch.save(model_log["tuned_llm"], root_dir+"/tuned_llm_model_log.pt") else: - model_log["tuned_llm"] = torch.load("tuned_llm_model_log.pt") + model_log["tuned_llm"] = torch.load(root_dir+"/tuned_llm_model_log.pt") del pure_llm gc.collect() @@ -355,7 +360,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode for name, Model, kwargs in zip(model_names, models, model_kwargs): model_log[name] = dict() train_model = True - if path.exists(f"{name}.pt"): + if path.exists(root_dir+f"/{name}.pt") and not force: print(f"Model {name} appears to already exist.") print("Would you like to retrain?") train_model = str(input("(y/n):")).lower() == "y" @@ -368,7 +373,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode since=since, num_epochs=epochs, hidden_channels=None, num_gnn_layers=None, batch_size=batch_size, eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, inference_fn=inference_fn, checkpointing=checkpointing, - tiny_llama=tiny_llama, dataset=dataset, model_save_name=name, model=model) + tiny_llama=tiny_llama, dataset=dataset, model_save_name=root_dir+'/'+name, model=model) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -378,15 +383,16 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode model_log[name]["eval_output"] = model_eval_outs print("E2E time (e2e_time) =", e2e_time, "seconds") print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") + model_log[name]["n_params"] = compute_n_parameters(model) del model gc.collect() else: - model_eval_outs = torch.load(f"{name}_eval_outs.pt") + model_eval_outs = torch.load(root_dir+f"/{name}_eval_outs.pt") # Calculate Hallucinations skip_hallucination_detection = False - if path.exists(f"{name}_model_log.pt"): + if path.exists(root_dir+f"/{name}_model_log.pt") and not force: print(f"Saved outputs for {name} have been found.") print("Would you like to redo?") user_input = str(input("(y/n):")).lower() @@ -397,15 +403,17 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode model_save_list = _eval_hallucinations_on_loader(model_eval_outs, loader, eval_batch_size) model_log[name]["hallucinates_list"] = model_save_list - torch.save(model_log[name], f"{name}_model_log.pt") + torch.save(model_log[name], root_dir+f"/{name}_model_log.pt") else: - model_log[name]["hallucinates_list"] = torch.load(f"{name}_model_log.pt")["hallucinates_list"] + model_log[name]["hallucinates_list"] = torch.load(root_dir+f"/{name}_model_log.pt")["hallucinates_list"] hal_dict = {k: [tup[1] for tup in v["hallucinates_list"]] for (k, v) in model_log.items()} - hallucinates_df = pd.DataFrame(hal_dict) - for col in hallucinates_df.columns: - print(f"Hallucinates for {col}:") - print(hallucinates_df[col].value_counts()) + hallucinates_df = pd.DataFrame(hal_dict).astype(str) + hallucinates_df = hallucinates_df.apply(pd.Series.value_counts).transpose() + hallucinates_df['e2e_time'] = pd.Series({k: v.get('e2e_time') for (k, v) in model_log.items()}) + hallucinates_df['n_params'] = pd.Series({k: v.get('n_params') for (k, v) in model_log.items()}) + print(hallucinates_df) + hallucinates_df.to_csv(root_dir+"/hallucinates_df.csv", index=False) def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 573434fc00fa..0a9182bdf858 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -61,6 +61,7 @@ def __init__( num_gnn_heads: int = 4, mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096, + mlp_out_tokens: int = 1, ) -> None: super().__init__() self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) @@ -115,9 +116,11 @@ def __init__( self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), - nn.Linear(mlp_hidden_dim, mlp_out_dim), + nn.Linear(mlp_hidden_dim, mlp_out_dim*mlp_out_tokens), ).to(self.llm_device) + self.mlp_out_tokens = mlp_out_tokens + self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, node_feat, edge_index, edge_attr, batch): From bf220d11d2ac54c10c5eda5e56ae7ee52eb38035 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 24 Jul 2024 13:33:17 -0700 Subject: [PATCH 642/752] override switch --- .../datasets/updated_web_qsp_dataset.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index e7438d74f703..9f12babe1c13 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -48,9 +48,10 @@ def retrieval_via_pcst( topk_e: int = 3, cost_e: float = 0.5, save_idx: bool = False, + override: bool = False, ) -> Tuple[Data, str]: c = 0.01 - if len(textual_nodes) == 0 or len(textual_edges) == 0: + if len(textual_nodes) == 0 or len(textual_edges) == 0 or override: desc = (textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( index=False, columns=["src", "edge_attr", "dst"])) @@ -232,6 +233,15 @@ def download(self) -> None: if self.limit >= 0: self.raw_dataset = self.raw_dataset.select(range(self.limit)) + self.split_idxs = { + "train": + torch.arange(self.limit//2), + "val": + torch.arange(self.limit//4) + self.limit//2, + "test": + torch.arange(self.limit//4) + self.limit//2 + + self.limit//4, + } self._save_raw_data() def _get_trips(self) -> Iterator[TripletLike]: @@ -287,6 +297,8 @@ def _retrieve_subgraphs(self) -> None: graph["node_idx"]].reset_index() textual_edges = self.textual_edges.iloc[ graph["edge_idx"]].reset_index() + pcst_subgraph = graph + ''' pcst_subgraph, desc = retrieval_via_pcst( graph, q_embs[index], @@ -296,12 +308,13 @@ def _retrieve_subgraphs(self) -> None: topk_e=5, cost_e=0.5, ) + ''' question = f"Question: {data_i['question']}\nAnswer: " label = ("|").join(data_i["answer"]).lower() pcst_subgraph["question"] = question pcst_subgraph["label"] = label - pcst_subgraph["desc"] = desc + #pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) print("Saving subgraphs...") self.save(list_of_graphs, self.processed_paths[0]) From c347994092718f83bc72f1d388a26cbd13c82255 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 24 Jul 2024 15:09:14 -0700 Subject: [PATCH 643/752] override switch 2 --- torch_geometric/datasets/updated_web_qsp_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 9f12babe1c13..57f031a293d7 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -172,9 +172,11 @@ def __init__( force_reload: bool = False, whole_graph_retrieval: bool = False, limit: int = -1, + override: bool = False, ) -> None: self.limit = limit self.whole_graph_retrieval = whole_graph_retrieval + self.override = override self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) @@ -297,8 +299,6 @@ def _retrieve_subgraphs(self) -> None: graph["node_idx"]].reset_index() textual_edges = self.textual_edges.iloc[ graph["edge_idx"]].reset_index() - pcst_subgraph = graph - ''' pcst_subgraph, desc = retrieval_via_pcst( graph, q_embs[index], @@ -307,14 +307,14 @@ def _retrieve_subgraphs(self) -> None: topk=3, topk_e=5, cost_e=0.5, + override=self.override, ) - ''' question = f"Question: {data_i['question']}\nAnswer: " label = ("|").join(data_i["answer"]).lower() pcst_subgraph["question"] = question pcst_subgraph["label"] = label - #pcst_subgraph["desc"] = desc + pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) print("Saving subgraphs...") self.save(list_of_graphs, self.processed_paths[0]) From 3611aa5021f4587e4643cf6713ce02f54a3d78fa Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 26 Jul 2024 17:10:03 -0700 Subject: [PATCH 644/752] rag generation script --- examples/llm_plus_gnn/rag_feature_store.py | 1 + examples/llm_plus_gnn/rag_generate.py | 75 ++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 examples/llm_plus_gnn/rag_generate.py diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index 0ca4c3de81ed..ca3a0ba646d4 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -89,4 +89,5 @@ def load_subgraph( class SentenceTransformerFeatureStore(KNNRAGFeatureStore): def __init__(self, *args, **kwargs): + kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py new file mode 100644 index 000000000000..d4bb628dec56 --- /dev/null +++ b/examples/llm_plus_gnn/rag_generate.py @@ -0,0 +1,75 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import RAGQueryLoader +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd + +# %% +ds = UpdatedWebQSPDataset("small_dataset", force_reload=True, limit=20) + +# %% +triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) + +# %% +questions = ds.raw_dataset['question'] +questions + +# %% +ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +num_edges = len(ds.indexer._edges) + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}) + +# %% +# Accuracy Metrics to be added to Profiler +def _eidx_helper(subg: Data, ground_truth: Data): + subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx + if isinstance(subg_eidx, torch.Tensor): + subg_eidx = subg_eidx.tolist() + if isinstance(gt_eidx, torch.Tensor): + gt_eidx = gt_eidx.tolist() + subg_e = set(subg_eidx) + gt_e = set(gt_eidx) + return subg_e, gt_e +def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + total_e = set(range(num_edges)) + tp = len(subg_e & gt_e) + tn = len(total_e-(subg_e | gt_e)) + return (tp+tn)/num_edges +def check_retrieval_precision(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(subg_e) +def check_retrieval_recall(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(gt_e) + +# %% +retrieval_stats = {"precision": [], "recall": [], "accuracy": []} +subgs = [] +for subg, gt in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): + retrieval_stats["precision"].append(check_retrieval_precision(subg, gt)) + retrieval_stats["recall"].append(check_retrieval_recall(subg, gt)) + retrieval_stats["accuracy"].append(check_retrieval_accuracy(subg, gt, num_edges)) + subgs.append(subg) + +pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') +torch.save(subgs, 'subg_results.pt') + From 953262f84c163bb86061f4c5313d942fcb2b3959 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 8 Aug 2024 14:34:50 -0700 Subject: [PATCH 645/752] wikidata dataset --- .../llm_plus_gnn/rag_generate_multihop.py | 38 + examples/llm_plus_gnn/wikidata.ipynb | 30776 ++++++++++++++++ 2 files changed, 30814 insertions(+) create mode 100644 examples/llm_plus_gnn/rag_generate_multihop.py create mode 100644 examples/llm_plus_gnn/wikidata.ipynb diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/rag_generate_multihop.py new file mode 100644 index 000000000000..3640dca39166 --- /dev/null +++ b/examples/llm_plus_gnn/rag_generate_multihop.py @@ -0,0 +1,38 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import RAGQueryLoader +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd + + +# %% +triplets = torch.load('wikimultihopqa_full_graph.pt') + +# %% +df = pd.read_csv('wikimultihopqa_cleaned.csv') +questions = df['question_text'][:10] + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=retrieval_via_pcst) + +# %% +subgs = [] +for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): + subgs.append(subg) + +torch.save(subgs, 'subg_results.pt') diff --git a/examples/llm_plus_gnn/wikidata.ipynb b/examples/llm_plus_gnn/wikidata.ipynb new file mode 100644 index 000000000000..a9e2c9b7bbc3 --- /dev/null +++ b/examples/llm_plus_gnn/wikidata.ipynb @@ -0,0 +1,30776 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import tqdm\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: Download dataset from this notebook instead of from outside it" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "lines = []\n", + "with open(\"/home/zaristei/repos/musique/raw_data/2wikimultihopqa_train_20k.jsonl\") as f:\n", + " for line in f:\n", + " lines.append(json.loads(line))\n", + "df_train = pd.DataFrame(lines)\n", + "df_train['is_train'] = True\n", + "lines = []\n", + "with open(\"/home/zaristei/repos/musique/raw_data/2wikimultihopqa_dev_20k.jsonl\") as f:\n", + " for line in f:\n", + " lines.append(json.loads(line))\n", + "df_test = pd.DataFrame(lines)\n", + "df_test['is_train'] = False\n", + "df = pd.concat([df_train, df_test])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "is_train\n", + "True 20000\n", + "False 12576\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['is_train'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "df['type'] = df['other_info'].apply(lambda row: row['type'])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "type\n", + "compositional 14339\n", + "comparison 9318\n", + "bridge_comparison 6834\n", + "inference 2085\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['type'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "df['graph'] = df['other_info'].apply(lambda row: row['evidences_id'])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "df['graph_size'] = df['graph'].apply(lambda row: len(row))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "graph_size\n", + "2 21390\n", + "0 8135\n", + "4 2540\n", + "5 328\n", + "3 183\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['graph_size'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "df = df[df['graph_size'] > 0]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "is_train\n", + "True 16377\n", + "False 8064\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['is_train'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['dataset', 'id', 'question_text', 'answer_text', 'answerable',\n", + " 'other_info', 'answer_aliases_texts', 'contexts',\n", + " 'decomposed_question_text', 'decomposed_instances',\n", + " 'component_question_texts', 'is_train', 'type', 'graph', 'graph_size'],\n", + " dtype='object')" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df = df[['id', 'question_text', 'answer_text', 'is_train', 'graph', 'type', 'graph_size']]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df.head()\n", + "refined_df.to_csv('wikimultihopqa_refined.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-08-07 16:56:03-- https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz?dl=1\n", + "Resolving www.dropbox.com (www.dropbox.com)... 162.125.13.18, 2620:100:6017:18::a27d:212\n", + "Connecting to www.dropbox.com (www.dropbox.com)|162.125.13.18|:443... connected.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://www.dropbox.com/scl/fi/jfqufhpsryt5sdfzaa63o/wikidata5m_alias.tar.gz?rlkey=hsy3o09yxmi5gpqpc2s4myzvh&dl=1 [following]\n", + "--2024-08-07 16:56:03-- https://www.dropbox.com/scl/fi/jfqufhpsryt5sdfzaa63o/wikidata5m_alias.tar.gz?rlkey=hsy3o09yxmi5gpqpc2s4myzvh&dl=1\n", + "Reusing existing connection to www.dropbox.com:443.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline/CYMQuDkY4mm8tz8BDJERS9ji8JJGNUFMRGZuSv3eQrSPj8ggW9-B__HKsfZoqM94GvbZoacWqN_QxovAJZ7c-EUuHhKPCD352hG7vSIYFBEI-5nwo4dmYbE5Ow1_XjRtsNLPJvygFKV_u-wpjzu6naI1/file?dl=1# [following]\n", + "--2024-08-07 16:56:04-- https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline/CYMQuDkY4mm8tz8BDJERS9ji8JJGNUFMRGZuSv3eQrSPj8ggW9-B__HKsfZoqM94GvbZoacWqN_QxovAJZ7c-EUuHhKPCD352hG7vSIYFBEI-5nwo4dmYbE5Ow1_XjRtsNLPJvygFKV_u-wpjzu6naI1/file?dl=1\n", + "Resolving uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com (uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com)... 162.125.13.15, 2620:100:601b:15::a27d:80f\n", + "Connecting to uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com (uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com)|162.125.13.15|:443... connected.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: /cd/0/inline2/CYMRQvw0HUqAAx3Y2uknr49kpFW0HvB1Qi9bPjKxqYAKIeFAkJ8x4G93uodurWf4UqOF3gW-_l3_evOuaRc5p3AYOCVzm3LhHUKPbMX4OW0ZvVa31yiNG0OGGBuZrgn8zVhc3aVpdhlcHUk6IGMK9CFY8uZL3-ZJfV-ckuXh_aAOuaT8aNXuHMCBWoTbiK2YAcJnWIog9ySrO3jxfLlV6QDHoR8gcJH91VePzZgDVkfG9ekXjO1Prbjkzyv8FPPpDyZtsPT9ysq8hbOmpGhwTCXQoApY7QhNG7bHtww4d9beY5O-e5UnaznYJqMbW6Dmc5Jz3zePlXVjAIG-DcnBvyfsQXeCGOO8ZD6f8WyxH3hKLwU0_sSQJqnUx9Kgu4Z5sng/file?dl=1 [following]\n", + "--2024-08-07 16:56:05-- https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline2/CYMRQvw0HUqAAx3Y2uknr49kpFW0HvB1Qi9bPjKxqYAKIeFAkJ8x4G93uodurWf4UqOF3gW-_l3_evOuaRc5p3AYOCVzm3LhHUKPbMX4OW0ZvVa31yiNG0OGGBuZrgn8zVhc3aVpdhlcHUk6IGMK9CFY8uZL3-ZJfV-ckuXh_aAOuaT8aNXuHMCBWoTbiK2YAcJnWIog9ySrO3jxfLlV6QDHoR8gcJH91VePzZgDVkfG9ekXjO1Prbjkzyv8FPPpDyZtsPT9ysq8hbOmpGhwTCXQoApY7QhNG7bHtww4d9beY5O-e5UnaznYJqMbW6Dmc5Jz3zePlXVjAIG-DcnBvyfsQXeCGOO8ZD6f8WyxH3hKLwU0_sSQJqnUx9Kgu4Z5sng/file?dl=1\n", + "Reusing existing connection to uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com:443.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 197450113 (188M) [application/binary]\n", + "Saving to: ‘wikidata5m_alias.tar.gz?dl=1’\n", + "\n", + "wikidata5m_alias.ta 100%[===================>] 188.30M 72.7MB/s in 2.6s \n", + "\n", + "2024-08-07 16:56:08 (72.7 MB/s) - ‘wikidata5m_alias.tar.gz?dl=1’ saved [197450113/197450113]\n", + "\n" + ] + } + ], + "source": [ + "!wget \"https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz?dl=1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "wikidata5m_entity.txt\n", + "wikidata5m_relation.txt\n" + ] + } + ], + "source": [ + "!tar -xvf \"wikidata5m_alias.tar.gz?dl=1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "P489\tcurrency symbol description\n", + "\n" + ] + } + ], + "source": [ + "with open('wikidata5m_relation.txt') as f:\n", + " print(f.readline())" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "825it [00:00, 193778.40it/s]\n" + ] + } + ], + "source": [ + "relation_map = {}\n", + "with open('wikidata5m_relation.txt') as f:\n", + " for line in tqdm.tqdm(f):\n", + " parts = line.strip().split('\\t')\n", + " for i in range(1, len(parts)):\n", + " if parts[i] not in relation_map:\n", + " relation_map[parts[i]] = []\n", + " relation_map[parts[i]].append(parts[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "inception\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "inception\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "inception\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "inception\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "editor ['P98', 'P1040']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "date of death\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "inception\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "date of birth\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "date of death\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "date of birth\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "country ['P3005', 'P17']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n", + "director ['P1037', 'P57']\n" + ] + } + ], + "source": [ + "# Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M\n", + "for row in refined_df.itertuples():\n", + " for trip in row.graph:\n", + " relation = trip[1]\n", + " if relation not in relation_map:\n", + " print(relation)\n", + " elif len(relation_map[relation]) > 1:\n", + " print(relation, relation_map[relation])" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "4813491it [00:04, 1109451.61it/s]\n" + ] + } + ], + "source": [ + "entity_set = set()\n", + "with open('wikidata5m_entity.txt') as f:\n", + " for line in tqdm.tqdm(f):\n", + " entity_set.add(line.strip().split('\\t')[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Q7069334\n", + "date_information\n", + "date_information\n", + "Q64768110\n", + "date_information\n", + "date_information\n", + "Q30324147\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63802341\n", + "date_information\n", + "date_information\n", + "Q3983360\n", + "date_information\n", + "date_information\n", + "Q60738284\n", + "date_information\n", + "date_information\n", + "Q12978262\n", + "date_information\n", + "date_information\n", + "Q3372419\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q59962784\n", + "date_information\n", + "date_information\n", + "Q3998173\n", + "date_information\n", + "Q28713010\n", + "Q331352\n", + "Q331352\n", + "Q54829004\n", + "date_information\n", + "Q12830714\n", + "date_information\n", + "Q60738580\n", + "date_information\n", + "Q19796493\n", + "Q39072549\n", + "Q63927168\n", + "date_information\n", + "date_information\n", + "Q1069403\n", + "Q51803959\n", + "date_information\n", + "date_information\n", + "Q65939060\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64174957\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18634612\n", + "date_information\n", + "Q4298967\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48672430\n", + "date_information\n", + "Q19694514\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48674696\n", + "Q48671501\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24483147\n", + "date_information\n", + "Q25106438\n", + "Q55081821\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65086758\n", + "Q2972533\n", + "Q24484294\n", + "Q3989858\n", + "date_information\n", + "date_information\n", + "Q56452642\n", + "date_information\n", + "Q3878817\n", + "date_information\n", + "date_information\n", + "Q64768127\n", + "Q54858959\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4428162\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4289253\n", + "date_information\n", + "Q17084224\n", + "date_information\n", + "Q1350154\n", + "date_information\n", + "date_information\n", + "Q64138712\n", + "Q76401690\n", + "Q3707073\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q3680696\n", + "Q3680696\n", + "date_information\n", + "date_information\n", + "Q63248839\n", + "date_information\n", + "Q54868320\n", + "date_information\n", + "date_information\n", + "Q60745215\n", + "Q50314058\n", + "Q3627346\n", + "date_information\n", + "date_information\n", + "Q3374809\n", + "Q3374809\n", + "Q64768701\n", + "Q992815\n", + "Q992815\n", + "Q15622114\n", + "Q28064471\n", + "Q60737563\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60456550\n", + "Q52151279\n", + "date_information\n", + "date_information\n", + "Q69827389\n", + "date_information\n", + "date_information\n", + "Q61958912\n", + "Q27\n", + "Q67130286\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58313464\n", + "date_information\n", + "Q56550941\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56284087\n", + "date_information\n", + "Q48672392\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60750745\n", + "Q18340884\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55403801\n", + "Q55403801\n", + "Q55606456\n", + "Q48673905\n", + "Q60737958\n", + "Q6214794\n", + "Q6214794\n", + "date_information\n", + "date_information\n", + "Q65924962\n", + "date_information\n", + "date_information\n", + "Q13788301\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q55595776\n", + "date_information\n", + "date_information\n", + "Q62129437\n", + "date_information\n", + "Q60738487\n", + "date_information\n", + "Q50819333\n", + "Q43388529\n", + "Q43388529\n", + "date_information\n", + "date_information\n", + "Q23814892\n", + "date_information\n", + "date_information\n", + "Q16309250\n", + "Q30898860\n", + "Q56241165\n", + "date_information\n", + "Q55419710\n", + "Q55419710\n", + "date_information\n", + "date_information\n", + "Q47090972\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19760848\n", + "Q48426793\n", + "Q34956353\n", + "date_information\n", + "Q63183587\n", + "date_information\n", + "Q61456874\n", + "Q45900448\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30623557\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3002150\n", + "Q23814892\n", + "Q4109853\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q59617591\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64577356\n", + "date_information\n", + "Q55102619\n", + "date_information\n", + "Q3742313\n", + "date_information\n", + "Q51885931\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64759209\n", + "Q2365600\n", + "Q57246562\n", + "Q15681455\n", + "date_information\n", + "date_information\n", + "Q64768836\n", + "date_information\n", + "date_information\n", + "Q6506487\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1068991\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65032686\n", + "date_information\n", + "date_information\n", + "Q60748551\n", + "date_information\n", + "Q56276953\n", + "date_information\n", + "Q11081421\n", + "date_information\n", + "Q21336011\n", + "Q56279611\n", + "date_information\n", + "Q12290762\n", + "Q3680696\n", + "Q3680696\n", + "date_information\n", + "date_information\n", + "Q53558909\n", + "Q2143090\n", + "Q6033589\n", + "Q6033589\n", + "Q56426852\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4844692\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q60549164\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3425284\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19329824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17079176\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48671589\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56152806\n", + "date_information\n", + "Q60738551\n", + "date_information\n", + "date_information\n", + "Q64577409\n", + "date_information\n", + "Q63969266\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q23976970\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q59728996\n", + "Q56279069\n", + "Q54868294\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47004710\n", + "date_information\n", + "Q64140895\n", + "date_information\n", + "date_information\n", + "Q10304581\n", + "Q10304581\n", + "date_information\n", + "Q20644750\n", + "Q76890341\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q76014955\n", + "Q2061015\n", + "Q64767767\n", + "date_information\n", + "Q51613741\n", + "Q27536286\n", + "Q56045065\n", + "date_information\n", + "Q63698633\n", + "Q17372365\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19884631\n", + "date_information\n", + "Q64876881\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65677528\n", + "Q17237979\n", + "Q17237979\n", + "date_information\n", + "date_information\n", + "Q17086825\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56063323\n", + "Q63802332\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63686494\n", + "Q55991302\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q5544317\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q57983083\n", + "Q55614153\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64355175\n", + "Q16310763\n", + "date_information\n", + "Q24189885\n", + "date_information\n", + "Q54020505\n", + "date_information\n", + "Q26921021\n", + "Q26921021\n", + "date_information\n", + "Q21168625\n", + "date_information\n", + "Q65086649\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12976077\n", + "Q9005\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q17124784\n", + "Q46992828\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20076298\n", + "Q1069296\n", + "Q47487574\n", + "date_information\n", + "Q4318541\n", + "Q51077490\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56692191\n", + "Q22043722\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3978256\n", + "Q3978256\n", + "date_information\n", + "date_information\n", + "Q19884760\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60315862\n", + "date_information\n", + "Q60738035\n", + "date_information\n", + "Q48674000\n", + "Q16568988\n", + "date_information\n", + "date_information\n", + "Q75279216\n", + "date_information\n", + "Q76488781\n", + "Q4063775\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q13629051\n", + "Q7014940\n", + "Q7014940\n", + "date_information\n", + "Q4449340\n", + "Q55607254\n", + "date_information\n", + "Q6060947\n", + "date_information\n", + "date_information\n", + "Q3825078\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3012352\n", + "Q18643972\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56086939\n", + "Q18160958\n", + "Q62020324\n", + "Q57314409\n", + "Q66764374\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47467250\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21639250\n", + "date_information\n", + "Q17084381\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65091853\n", + "Q14333308\n", + "date_information\n", + "Q1057286\n", + "date_information\n", + "Q25104948\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63969278\n", + "Q31734750\n", + "date_information\n", + "date_information\n", + "Q48772026\n", + "date_information\n", + "Q3680696\n", + "Q3680696\n", + "Q50650139\n", + "Q45040758\n", + "Q48673974\n", + "date_information\n", + "date_information\n", + "Q56042875\n", + "Q19884834\n", + "date_information\n", + "date_information\n", + "Q55806689\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q456383\n", + "Q456383\n", + "date_information\n", + "date_information\n", + "Q4184537\n", + "date_information\n", + "Q55768557\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2781903\n", + "Q2781903\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65032573\n", + "date_information\n", + "date_information\n", + "Q4459376\n", + "Q55621958\n", + "date_information\n", + "date_information\n", + "Q2157653\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q5706176\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56703335\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q331352\n", + "Q331352\n", + "Q63802265\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51955042\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24261792\n", + "Q59393445\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62023950\n", + "date_information\n", + "Q72422839\n", + "date_information\n", + "date_information\n", + "Q65032648\n", + "Q66305376\n", + "Q55227056\n", + "Q55227056\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q43398827\n", + "Q2570066\n", + "Q4131248\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q10655046\n", + "date_information\n", + "Q60790967\n", + "date_information\n", + "date_information\n", + "Q18554460\n", + "Q48671922\n", + "Q48757939\n", + "Q1062145\n", + "Q25505760\n", + "Q4365301\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q5253994\n", + "Q27530932\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25505760\n", + "Q28457976\n", + "date_information\n", + "date_information\n", + "Q46230954\n", + "Q46230954\n", + "Q16625133\n", + "date_information\n", + "Q64759208\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30312756\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q71050994\n", + "Q59167929\n", + "Q47500739\n", + "date_information\n", + "date_information\n", + "Q75274558\n", + "date_information\n", + "Q19690966\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q72280034\n", + "date_information\n", + "date_information\n", + "Q63994392\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55641238\n", + "Q56063528\n", + "date_information\n", + "date_information\n", + "Q11983511\n", + "date_information\n", + "date_information\n", + "Q65677545\n", + "date_information\n", + "Q17478875\n", + "date_information\n", + "date_information\n", + "Q48798236\n", + "Q60737736\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58214420\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63924529\n", + "Q62595482\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q48674262\n", + "date_information\n", + "Q57584335\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65052620\n", + "date_information\n", + "Q60737570\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47488371\n", + "Q76014955\n", + "Q28434505\n", + "Q66305376\n", + "date_information\n", + "date_information\n", + "Q55317987\n", + "date_information\n", + "Q55363997\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64211107\n", + "date_information\n", + "date_information\n", + "Q61450772\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15143923\n", + "Q2159814\n", + "Q48671626\n", + "Q56284087\n", + "Q51540879\n", + "date_information\n", + "date_information\n", + "Q23900134\n", + "date_information\n", + "Q75276877\n", + "Q64768050\n", + "Q61365850\n", + "Q61365850\n", + "date_information\n", + "Q1049634\n", + "date_information\n", + "Q48671205\n", + "date_information\n", + "date_information\n", + "Q64018111\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62003783\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q10388675\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q49700441\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60062917\n", + "date_information\n", + "Q57328748\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4171785\n", + "Q60737762\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19374242\n", + "date_information\n", + "Q60637519\n", + "date_information\n", + "date_information\n", + "Q5438207\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55333388\n", + "date_information\n", + "Q55165507\n", + "Q55081823\n", + "date_information\n", + "date_information\n", + "Q50295992\n", + "Q3808084\n", + "date_information\n", + "Q22075101\n", + "date_information\n", + "date_information\n", + "Q12232417\n", + "Q30633631\n", + "Q63924529\n", + "Q2907144\n", + "date_information\n", + "date_information\n", + "Q11161132\n", + "Q55772702\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62277831\n", + "Q71246482\n", + "date_information\n", + "date_information\n", + "Q56615611\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21336011\n", + "Q62018327\n", + "date_information\n", + "Q27\n", + "Q45763740\n", + "date_information\n", + "Q55612903\n", + "Q60767894\n", + "Q19887333\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1087285\n", + "Q3321378\n", + "Q64768647\n", + "Q2282608\n", + "date_information\n", + "date_information\n", + "Q78305911\n", + "date_information\n", + "Q3304810\n", + "Q15270573\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63698573\n", + "date_information\n", + "date_information\n", + "Q60745185\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q45046451\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3567855\n", + "Q7936508\n", + "Q7936508\n", + "date_information\n", + "date_information\n", + "Q60738633\n", + "Q64627115\n", + "date_information\n", + "Q16614474\n", + "date_information\n", + "Q56322655\n", + "Q52715786\n", + "date_information\n", + "Q3419202\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65678415\n", + "Q66124718\n", + "Q66124718\n", + "date_information\n", + "date_information\n", + "Q62860838\n", + "Q62860838\n", + "Q63986511\n", + "Q63986512\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15706063\n", + "Q3958334\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12988947\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24659886\n", + "Q75521509\n", + "Q26935570\n", + "Q48790611\n", + "Q4378422\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48674657\n", + "Q55647759\n", + "date_information\n", + "Q65080213\n", + "Q66688160\n", + "Q58814919\n", + "Q76273253\n", + "Q19908291\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58814888\n", + "date_information\n", + "Q60738293\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56401290\n", + "date_information\n", + "Q3549581\n", + "Q56596744\n", + "date_information\n", + "date_information\n", + "Q10406203\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19618477\n", + "date_information\n", + "date_information\n", + "Q13009424\n", + "date_information\n", + "Q75249592\n", + "date_information\n", + "Q56045974\n", + "Q48835993\n", + "Q23073331\n", + "Q55767505\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q19952744\n", + "Q63450254\n", + "date_information\n", + "date_information\n", + "Q16513344\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q59728996\n", + "Q51955042\n", + "date_information\n", + "Q52158771\n", + "Q63398302\n", + "Q55450652\n", + "Q27\n", + "Q44485312\n", + "Q60737553\n", + "Q75804226\n", + "date_information\n", + "date_information\n", + "Q56416462\n", + "date_information\n", + "Q75253659\n", + "date_information\n", + "date_information\n", + "Q60197034\n", + "date_information\n", + "date_information\n", + "Q15905014\n", + "date_information\n", + "date_information\n", + "Q63969256\n", + "date_information\n", + "Q10322276\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18405892\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50650382\n", + "date_information\n", + "Q76014955\n", + "date_information\n", + "Q3874453\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30621521\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60737691\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66828708\n", + "date_information\n", + "Q75497811\n", + "Q63565874\n", + "date_information\n", + "date_information\n", + "Q11048250\n", + "date_information\n", + "Q1061054\n", + "date_information\n", + "date_information\n", + "Q4130286\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q76186239\n", + "Q18518927\n", + "Q18518927\n", + "date_information\n", + "Q28406793\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q27\n", + "Q58815150\n", + "date_information\n", + "Q18364314\n", + "date_information\n", + "Q60734621\n", + "date_information\n", + "date_information\n", + "Q75492515\n", + "Q64549196\n", + "date_information\n", + "Q63119071\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16028461\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19655285\n", + "date_information\n", + "date_information\n", + "Q19618531\n", + "Q56816889\n", + "date_information\n", + "Q62075985\n", + "date_information\n", + "Q73538385\n", + "date_information\n", + "date_information\n", + "Q12976368\n", + "Q29912823\n", + "date_information\n", + "date_information\n", + "Q15689850\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24940239\n", + "date_information\n", + "Q24729251\n", + "Q63246549\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64485310\n", + "Q15623497\n", + "Q20005025\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56416519\n", + "date_information\n", + "Q59917796\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77885110\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q46600570\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58411267\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47004710\n", + "Q1388181\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75574603\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63924479\n", + "Q64460079\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55726186\n", + "date_information\n", + "date_information\n", + "Q30880265\n", + "date_information\n", + "date_information\n", + "Q54020341\n", + "Q48672538\n", + "date_information\n", + "date_information\n", + "Q48674649\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55590066\n", + "date_information\n", + "Q66363590\n", + "Q3522860\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56703332\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20093154\n", + "Q65082453\n", + "date_information\n", + "Q65320559\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55637523\n", + "Q47500749\n", + "date_information\n", + "Q60165041\n", + "Q4076139\n", + "date_information\n", + "Q61450811\n", + "Q65678838\n", + "date_information\n", + "Q24728788\n", + "Q63183577\n", + "Q6966375\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47004852\n", + "date_information\n", + "Q11982970\n", + "Q77892959\n", + "date_information\n", + "Q21337743\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25610585\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q8557599\n", + "date_information\n", + "Q47455556\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60783304\n", + "date_information\n", + "Q12977047\n", + "Q66588063\n", + "date_information\n", + "Q19884740\n", + "Q2929293\n", + "date_information\n", + "Q60463503\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4108587\n", + "date_information\n", + "Q24672337\n", + "Q24672337\n", + "Q12943321\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50728314\n", + "Q4078557\n", + "Q58754776\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q77892476\n", + "date_information\n", + "date_information\n", + "Q53674897\n", + "Q3018197\n", + "Q3018197\n", + "date_information\n", + "Q55393988\n", + "date_information\n", + "Q13640080\n", + "date_information\n", + "Q28457675\n", + "date_information\n", + "date_information\n", + "Q12008229\n", + "Q60764980\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64460234\n", + "date_information\n", + "Q5675398\n", + "Q48674553\n", + "Q27703697\n", + "Q64768590\n", + "date_information\n", + "Q48671619\n", + "date_information\n", + "date_information\n", + "Q64768319\n", + "date_information\n", + "date_information\n", + "Q66840990\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q64768648\n", + "Q16016727\n", + "Q74193018\n", + "date_information\n", + "Q57053277\n", + "Q57053277\n", + "date_information\n", + "date_information\n", + "Q331352\n", + "Q331352\n", + "Q62073359\n", + "Q56279425\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3754764\n", + "date_information\n", + "date_information\n", + "Q21852592\n", + "date_information\n", + "Q56063218\n", + "date_information\n", + "date_information\n", + "Q1913674\n", + "date_information\n", + "Q1475421\n", + "date_information\n", + "Q50755863\n", + "Q28457524\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52151200\n", + "Q65064897\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q10791093\n", + "Q55258973\n", + "Q3289992\n", + "date_information\n", + "Q997387\n", + "Q997387\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1056743\n", + "Q27\n", + "Q64830343\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q29912823\n", + "date_information\n", + "Q16134448\n", + "Q16134448\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60737621\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q41787227\n", + "Q56279002\n", + "date_information\n", + "date_information\n", + "Q17199295\n", + "date_information\n", + "date_information\n", + "Q22978862\n", + "date_information\n", + "Q2625914\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3213381\n", + "date_information\n", + "Q55621609\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q10952924\n", + "date_information\n", + "Q79120099\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q39075975\n", + "date_information\n", + "Q50450145\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q16440157\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1054847\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12930727\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19278235\n", + "Q60737632\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21843347\n", + "date_information\n", + "date_information\n", + "Q19521612\n", + "date_information\n", + "date_information\n", + "Q58437727\n", + "date_information\n", + "date_information\n", + "Q56537271\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2940176\n", + "date_information\n", + "Q55611501\n", + "Q60737676\n", + "Q60738269\n", + "date_information\n", + "Q4108743\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56289818\n", + "date_information\n", + "Q21343277\n", + "date_information\n", + "Q2900415\n", + "date_information\n", + "Q64755607\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28147010\n", + "date_information\n", + "date_information\n", + "Q19387802\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16028548\n", + "Q77892399\n", + "Q44821118\n", + "Q59305418\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4340096\n", + "date_information\n", + "date_information\n", + "Q4207072\n", + "Q4296236\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65061089\n", + "date_information\n", + "Q47452391\n", + "Q19907507\n", + "Q55907550\n", + "Q52145034\n", + "Q66764383\n", + "Q60738164\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16253118\n", + "date_information\n", + "Q16015511\n", + "Q60646525\n", + "date_information\n", + "Q54868306\n", + "date_information\n", + "date_information\n", + "Q56452637\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66841330\n", + "Q54020757\n", + "date_information\n", + "Q47450881\n", + "date_information\n", + "Q47081778\n", + "date_information\n", + "Q60439387\n", + "date_information\n", + "Q57584371\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52276380\n", + "Q65087828\n", + "date_information\n", + "date_information\n", + "Q17416360\n", + "date_information\n", + "Q12977309\n", + "date_information\n", + "Q38049163\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50074317\n", + "date_information\n", + "Q50414212\n", + "Q27\n", + "Q2302896\n", + "Q2302896\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3429285\n", + "Q4237108\n", + "Q56241172\n", + "date_information\n", + "Q55552345\n", + "date_information\n", + "Q56703329\n", + "Q52715793\n", + "date_information\n", + "Q40745153\n", + "date_information\n", + "date_information\n", + "Q592788\n", + "date_information\n", + "Q3192937\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48996920\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21040363\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30594579\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64174444\n", + "date_information\n", + "Q54879127\n", + "date_information\n", + "date_information\n", + "Q23655553\n", + "Q23655553\n", + "date_information\n", + "date_information\n", + "Q55611897\n", + "date_information\n", + "date_information\n", + "Q31310887\n", + "Q65045015\n", + "Q3221009\n", + "Q52776612\n", + "Q64768124\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28220176\n", + "date_information\n", + "date_information\n", + "Q2590303\n", + "Q12991096\n", + "date_information\n", + "Q1070043\n", + "date_information\n", + "date_information\n", + "Q63119112\n", + "Q51730226\n", + "Q51730226\n", + "date_information\n", + "Q47034913\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25505742\n", + "Q15721083\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63924372\n", + "Q62631136\n", + "Q47307336\n", + "date_information\n", + "date_information\n", + "Q53854471\n", + "date_information\n", + "date_information\n", + "Q30633641\n", + "Q13570759\n", + "Q3749311\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56037055\n", + "Q58843878\n", + "Q48815037\n", + "date_information\n", + "Q1062023\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12439936\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18086419\n", + "Q56452620\n", + "date_information\n", + "Q56692195\n", + "date_information\n", + "Q16316783\n", + "date_information\n", + "Q63020397\n", + "Q45192617\n", + "Q45192617\n", + "date_information\n", + "Q8851702\n", + "date_information\n", + "Q55612948\n", + "date_information\n", + "Q65081567\n", + "Q56416519\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56275844\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2326685\n", + "Q18554460\n", + "date_information\n", + "date_information\n", + "Q47513340\n", + "date_information\n", + "Q48784694\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51077300\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21183743\n", + "date_information\n", + "Q19329824\n", + "Q27\n", + "Q49239599\n", + "date_information\n", + "Q50825750\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q53374241\n", + "date_information\n", + "date_information\n", + "Q16633939\n", + "date_information\n", + "Q47064237\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738635\n", + "Q64768137\n", + "Q69629123\n", + "date_information\n", + "Q77892368\n", + "date_information\n", + "Q47455814\n", + "Q63247492\n", + "Q60738560\n", + "date_information\n", + "Q55633963\n", + "Q21081425\n", + "Q50825136\n", + "Q50825136\n", + "date_information\n", + "date_information\n", + "Q27535196\n", + "Q65073215\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4102998\n", + "date_information\n", + "Q56281825\n", + "date_information\n", + "date_information\n", + "Q65924942\n", + "Q67146474\n", + "Q47034953\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24261792\n", + "date_information\n", + "date_information\n", + "Q3858110\n", + "date_information\n", + "Q55450595\n", + "Q56600254\n", + "date_information\n", + "date_information\n", + "Q66691283\n", + "date_information\n", + "date_information\n", + "Q74190856\n", + "date_information\n", + "Q3025000\n", + "date_information\n", + "date_information\n", + "Q55637599\n", + "Q3521000\n", + "date_information\n", + "Q64211103\n", + "Q30738154\n", + "Q48671816\n", + "date_information\n", + "Q25619745\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q43384713\n", + "date_information\n", + "date_information\n", + "Q56279529\n", + "Q47063859\n", + "Q47455556\n", + "date_information\n", + "Q17082113\n", + "date_information\n", + "date_information\n", + "Q17079363\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54229562\n", + "Q19907720\n", + "date_information\n", + "date_information\n", + "Q47038485\n", + "date_information\n", + "Q43303325\n", + "date_information\n", + "date_information\n", + "Q55623763\n", + "Q55623763\n", + "date_information\n", + "Q57780001\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q67222754\n", + "Q65071895\n", + "date_information\n", + "Q55696425\n", + "date_information\n", + "Q11634137\n", + "date_information\n", + "Q3986259\n", + "Q55615148\n", + "Q55623864\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56651740\n", + "Q62278093\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q63698666\n", + "date_information\n", + "date_information\n", + "Q46606647\n", + "Q56249739\n", + "Q64338322\n", + "Q56249739\n", + "Q66459265\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1192808\n", + "Q4246432\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64577360\n", + "Q56405840\n", + "Q15633656\n", + "date_information\n", + "date_information\n", + "Q5668844\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16679535\n", + "date_information\n", + "Q7377305\n", + "Q4199413\n", + "Q63735869\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63001772\n", + "date_information\n", + "Q60191520\n", + "Q15926680\n", + "date_information\n", + "Q64140888\n", + "date_information\n", + "Q6161423\n", + "date_information\n", + "date_information\n", + "Q25738584\n", + "date_information\n", + "Q48672646\n", + "Q54021017\n", + "Q62631150\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q55625195\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18518927\n", + "date_information\n", + "date_information\n", + "Q56276591\n", + "Q73536300\n", + "date_information\n", + "Q61972627\n", + "date_information\n", + "Q28690154\n", + "date_information\n", + "Q60310526\n", + "date_information\n", + "Q48816553\n", + "date_information\n", + "date_information\n", + "Q25539787\n", + "Q31295811\n", + "date_information\n", + "Q24484294\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60683565\n", + "Q60690251\n", + "Q60690251\n", + "Q21672570\n", + "Q65679595\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64759228\n", + "Q48674397\n", + "date_information\n", + "Q48673900\n", + "date_information\n", + "Q4508339\n", + "Q52151214\n", + "Q12707541\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56045021\n", + "date_information\n", + "Q3192937\n", + "Q61037667\n", + "date_information\n", + "Q60738529\n", + "date_information\n", + "Q3829140\n", + "Q3849100\n", + "Q3849100\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63565887\n", + "date_information\n", + "date_information\n", + "Q52715614\n", + "date_information\n", + "Q12974353\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738058\n", + "date_information\n", + "date_information\n", + "Q44492063\n", + "Q69629123\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768391\n", + "Q52715740\n", + "Q4210161\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48805436\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55627581\n", + "Q55598788\n", + "Q20978738\n", + "Q20978738\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48739903\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52320451\n", + "date_information\n", + "date_information\n", + "Q38251170\n", + "Q3724386\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63248850\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20245404\n", + "Q48671499\n", + "Q51285350\n", + "Q75742719\n", + "date_information\n", + "date_information\n", + "Q30738154\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12924557\n", + "date_information\n", + "date_information\n", + "Q60738403\n", + "Q60738268\n", + "date_information\n", + "Q77895522\n", + "date_information\n", + "date_information\n", + "Q61669648\n", + "date_information\n", + "Q55632769\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25508485\n", + "Q55611631\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17080137\n", + "date_information\n", + "Q19886924\n", + "date_information\n", + "Q55278849\n", + "Q47542267\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q46600495\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65057861\n", + "Q27\n", + "Q45710417\n", + "date_information\n", + "date_information\n", + "Q63372375\n", + "date_information\n", + "date_information\n", + "Q47011389\n", + "date_information\n", + "Q24728189\n", + "date_information\n", + "Q60745160\n", + "Q55616544\n", + "date_information\n", + "date_information\n", + "Q61747353\n", + "Q28509912\n", + "date_information\n", + "date_information\n", + "Q56604157\n", + "Q61896055\n", + "date_information\n", + "date_information\n", + "Q61747347\n", + "date_information\n", + "date_information\n", + "Q64708093\n", + "Q16664499\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q46369290\n", + "Q60875748\n", + "date_information\n", + "Q4478405\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55609409\n", + "Q64848293\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738412\n", + "date_information\n", + "Q64939837\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55631173\n", + "Q60848339\n", + "Q55635810\n", + "Q55758066\n", + "date_information\n", + "Q19946737\n", + "date_information\n", + "date_information\n", + "Q62595516\n", + "Q23014197\n", + "Q47528044\n", + "Q60787303\n", + "date_information\n", + "Q27963551\n", + "date_information\n", + "Q66364883\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q18713310\n", + "Q55694395\n", + "date_information\n", + "date_information\n", + "Q4319017\n", + "date_information\n", + "date_information\n", + "Q48672721\n", + "Q6503488\n", + "date_information\n", + "Q24153864\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55946142\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54868325\n", + "date_information\n", + "Q48970825\n", + "Q48671199\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50309110\n", + "Q44606399\n", + "date_information\n", + "Q13640271\n", + "date_information\n", + "date_information\n", + "Q59149494\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65087828\n", + "date_information\n", + "Q59692468\n", + "Q56048936\n", + "Q19613178\n", + "date_information\n", + "Q64338245\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48988129\n", + "date_information\n", + "date_information\n", + "Q48671581\n", + "date_information\n", + "date_information\n", + "Q48684414\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4211954\n", + "Q15224621\n", + "date_information\n", + "Q45899823\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55605752\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19614422\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56070897\n", + "date_information\n", + "Q19614201\n", + "Q1052809\n", + "Q22917273\n", + "date_information\n", + "Q60737739\n", + "date_information\n", + "Q1568748\n", + "date_information\n", + "Q56274437\n", + "Q15514334\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63565879\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60748445\n", + "date_information\n", + "Q17478936\n", + "date_information\n", + "date_information\n", + "Q64768489\n", + "Q52063072\n", + "date_information\n", + "date_information\n", + "Q3482558\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q72380423\n", + "Q60734654\n", + "Q26213102\n", + "date_information\n", + "date_information\n", + "Q27536269\n", + "date_information\n", + "date_information\n", + "Q1053778\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61664915\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60565417\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q6172033\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60487164\n", + "date_information\n", + "Q29643416\n", + "date_information\n", + "Q20245957\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24975344\n", + "date_information\n", + "Q47009365\n", + "date_information\n", + "Q65678112\n", + "date_information\n", + "date_information\n", + "Q55635268\n", + "date_information\n", + "Q64768620\n", + "date_information\n", + "date_information\n", + "Q1285133\n", + "date_information\n", + "Q56278697\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q70887035\n", + "Q53717659\n", + "Q15635899\n", + "Q15635899\n", + "Q7881655\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q45125179\n", + "date_information\n", + "Q48671573\n", + "Q56598902\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54829004\n", + "Q10441356\n", + "Q64768153\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60370884\n", + "Q60785708\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q53912009\n", + "Q54819554\n", + "Q63450179\n", + "Q7271631\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q65051886\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47468248\n", + "date_information\n", + "date_information\n", + "Q4009651\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63643994\n", + "Q47500766\n", + "Q4053879\n", + "Q60738710\n", + "date_information\n", + "Q62031912\n", + "date_information\n", + "date_information\n", + "Q62277405\n", + "date_information\n", + "date_information\n", + "Q51999611\n", + "date_information\n", + "Q17639864\n", + "date_information\n", + "Q331352\n", + "Q331352\n", + "date_information\n", + "date_information\n", + "Q65090819\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3522853\n", + "Q4193938\n", + "date_information\n", + "Q49885354\n", + "date_information\n", + "date_information\n", + "Q12985827\n", + "date_information\n", + "Q55258983\n", + "date_information\n", + "Q57314412\n", + "date_information\n", + "Q1631673\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3296267\n", + "date_information\n", + "Q48674522\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56416519\n", + "Q18643972\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q7936508\n", + "Q7936508\n", + "Q56092489\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56615611\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52157923\n", + "date_information\n", + "Q60738619\n", + "Q54021044\n", + "date_information\n", + "date_information\n", + "Q55635199\n", + "date_information\n", + "Q19619138\n", + "Q64768680\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4213798\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4378254\n", + "date_information\n", + "Q62595390\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48804187\n", + "Q48674213\n", + "Q66305379\n", + "Q3429285\n", + "date_information\n", + "Q3670793\n", + "Q4289625\n", + "date_information\n", + "date_information\n", + "Q3986504\n", + "Q67266090\n", + "Q33170323\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12537376\n", + "Q53613358\n", + "date_information\n", + "date_information\n", + "Q55759051\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20245957\n", + "date_information\n", + "date_information\n", + "Q18500792\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1056331\n", + "Q11884841\n", + "Q64768732\n", + "Q65065675\n", + "date_information\n", + "Q31192565\n", + "Q1067104\n", + "Q59124886\n", + "Q59124886\n", + "date_information\n", + "Q30684118\n", + "date_information\n", + "Q51278968\n", + "date_information\n", + "date_information\n", + "Q27527841\n", + "Q4255603\n", + "Q62018324\n", + "date_information\n", + "date_information\n", + "Q56284302\n", + "date_information\n", + "date_information\n", + "Q77896663\n", + "Q56452587\n", + "Q3314521\n", + "Q3314521\n", + "date_information\n", + "Q27\n", + "Q4734672\n", + "date_information\n", + "Q12991152\n", + "date_information\n", + "date_information\n", + "Q65924938\n", + "date_information\n", + "Q60737546\n", + "date_information\n", + "Q48318033\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q6418615\n", + "Q6418615\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q13564944\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4199990\n", + "date_information\n", + "date_information\n", + "Q17478107\n", + "date_information\n", + "date_information\n", + "Q48672665\n", + "date_information\n", + "Q16557779\n", + "Q1068486\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2163100\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q50649657\n", + "date_information\n", + "date_information\n", + "Q56045047\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q8837872\n", + "date_information\n", + "Q59617591\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63398838\n", + "date_information\n", + "Q65679587\n", + "date_information\n", + "date_information\n", + "Q50650382\n", + "Q65552652\n", + "Q66455732\n", + "date_information\n", + "date_information\n", + "Q67147057\n", + "Q56291009\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4742176\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63869121\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1024824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3422649\n", + "Q21342153\n", + "date_information\n", + "Q15224531\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56816874\n", + "date_information\n", + "Q47455556\n", + "date_information\n", + "date_information\n", + "Q65065129\n", + "Q25490712\n", + "Q54020295\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15702124\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58814916\n", + "Q50649795\n", + "date_information\n", + "Q55583728\n", + "Q4423129\n", + "date_information\n", + "date_information\n", + "Q2571155\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q63565895\n", + "date_information\n", + "Q62018336\n", + "date_information\n", + "date_information\n", + "Q75278776\n", + "date_information\n", + "Q11339934\n", + "Q19618217\n", + "date_information\n", + "Q20004254\n", + "Q56071282\n", + "Q48818141\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61750746\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18554460\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q11824526\n", + "Q30901566\n", + "date_information\n", + "Q19896174\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56248006\n", + "Q56248006\n", + "date_information\n", + "date_information\n", + "Q5659177\n", + "Q16719675\n", + "date_information\n", + "Q63923824\n", + "date_information\n", + "Q49750686\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25396637\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19848388\n", + "Q331352\n", + "Q331352\n", + "Q45302060\n", + "Q11993882\n", + "Q53854471\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56756930\n", + "date_information\n", + "Q55762020\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15615493\n", + "date_information\n", + "date_information\n", + "Q4313070\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51278938\n", + "Q60738710\n", + "date_information\n", + "Q48835993\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738182\n", + "Q64577408\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60740385\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1583694\n", + "Q16766207\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48673905\n", + "Q56886895\n", + "date_information\n", + "Q55625517\n", + "date_information\n", + "Q66498260\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4298967\n", + "Q64597510\n", + "date_information\n", + "Q29168841\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48422401\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77906377\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q39058429\n", + "Q46840692\n", + "Q47091822\n", + "Q47091822\n", + "date_information\n", + "date_information\n", + "Q54020520\n", + "Q48671517\n", + "Q3749740\n", + "Q56678187\n", + "date_information\n", + "Q2819749\n", + "date_information\n", + "Q55633312\n", + "Q55605990\n", + "Q56249739\n", + "Q56249739\n", + "date_information\n", + "Q17478906\n", + "date_information\n", + "Q37103689\n", + "Q64768157\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q31899662\n", + "date_information\n", + "Q25505901\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16614618\n", + "date_information\n", + "Q56291196\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q45710167\n", + "Q55629777\n", + "Q21672950\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q63248839\n", + "date_information\n", + "Q47455556\n", + "date_information\n", + "Q17154585\n", + "date_information\n", + "Q11920998\n", + "Q55607176\n", + "date_information\n", + "Q68681420\n", + "date_information\n", + "Q4189600\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q44506078\n", + "Q27\n", + "date_information\n", + "Q60737759\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q8826702\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65072188\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15805124\n", + "Q77892368\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q46606577\n", + "date_information\n", + "Q16028548\n", + "Q56274942\n", + "date_information\n", + "date_information\n", + "Q51031293\n", + "date_information\n", + "Q2583655\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q57773022\n", + "Q63969266\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56458840\n", + "date_information\n", + "date_information\n", + "Q60738177\n", + "Q59733171\n", + "date_information\n", + "Q56614298\n", + "Q49114543\n", + "Q56886875\n", + "date_information\n", + "date_information\n", + "Q22000108\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65045109\n", + "date_information\n", + "Q56452546\n", + "Q56334062\n", + "Q3824783\n", + "Q56334062\n", + "Q60738269\n", + "Q3563923\n", + "Q18471775\n", + "Q3563923\n", + "Q4278661\n", + "Q51031341\n", + "Q5990904\n", + "Q51756485\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56537271\n", + "date_information\n", + "date_information\n", + "Q55635194\n", + "date_information\n", + "Q47500692\n", + "date_information\n", + "Q62277379\n", + "date_information\n", + "Q58814919\n", + "date_information\n", + "Q3798546\n", + "Q55098902\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56045065\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56063974\n", + "date_information\n", + "date_information\n", + "Q77903976\n", + "Q64768676\n", + "date_information\n", + "date_information\n", + "Q2134675\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18554460\n", + "Q56874855\n", + "Q51879925\n", + "date_information\n", + "Q66828800\n", + "Q12991096\n", + "Q19848394\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62278019\n", + "Q19329824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64691762\n", + "date_information\n", + "Q52151197\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56615750\n", + "Q12930727\n", + "Q64338322\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64338187\n", + "date_information\n", + "date_information\n", + "Q18406892\n", + "Q19916458\n", + "date_information\n", + "Q57983086\n", + "date_information\n", + "Q43401872\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3990127\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51834363\n", + "date_information\n", + "Q48671238\n", + "Q55637590\n", + "Q64221113\n", + "date_information\n", + "Q48817737\n", + "date_information\n", + "Q53036569\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48671709\n", + "date_information\n", + "Q64768132\n", + "Q65058947\n", + "date_information\n", + "date_information\n", + "Q28466514\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q67628998\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65087118\n", + "Q12002018\n", + "date_information\n", + "Q48671879\n", + "date_information\n", + "Q59836204\n", + "date_information\n", + "date_information\n", + "Q9094439\n", + "date_information\n", + "date_information\n", + "Q2657975\n", + "Q3986523\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q60737958\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56886874\n", + "Q56703333\n", + "Q27\n", + "Q56276944\n", + "Q64768061\n", + "Q63450091\n", + "Q48672731\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19907410\n", + "date_information\n", + "Q3208390\n", + "date_information\n", + "Q55609415\n", + "Q64577347\n", + "Q48990162\n", + "Q48990162\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19613400\n", + "Q24175518\n", + "Q77895479\n", + "date_information\n", + "Q12205404\n", + "Q60738280\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48898535\n", + "date_information\n", + "Q60727937\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q79212167\n", + "date_information\n", + "Q56279327\n", + "Q64211126\n", + "Q44727126\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24728755\n", + "date_information\n", + "date_information\n", + "Q25738039\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65091022\n", + "date_information\n", + "Q65491495\n", + "date_information\n", + "date_information\n", + "Q59728996\n", + "Q16592335\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58094575\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q6560644\n", + "date_information\n", + "date_information\n", + "Q47038485\n", + "date_information\n", + "Q58310549\n", + "date_information\n", + "Q21335696\n", + "Q3192937\n", + "date_information\n", + "date_information\n", + "Q1062924\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q47455556\n", + "Q44417422\n", + "Q61510092\n", + "date_information\n", + "date_information\n", + "Q48672572\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3010170\n", + "date_information\n", + "date_information\n", + "Q56275660\n", + "date_information\n", + "Q24484703\n", + "Q66764359\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52151277\n", + "Q46152071\n", + "date_information\n", + "date_information\n", + "Q25447892\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q11824571\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1050472\n", + "date_information\n", + "Q31189596\n", + "date_information\n", + "Q4297640\n", + "Q48770599\n", + "Q16679476\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30738154\n", + "Q16315884\n", + "date_information\n", + "Q62761097\n", + "date_information\n", + "Q48965733\n", + "date_information\n", + "Q55390546\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51756485\n", + "date_information\n", + "Q55621373\n", + "date_information\n", + "date_information\n", + "Q1075917\n", + "Q48751380\n", + "date_information\n", + "date_information\n", + "Q48674142\n", + "date_information\n", + "Q48733721\n", + "Q48733721\n", + "Q77906097\n", + "Q58411258\n", + "Q49700441\n", + "Q26259702\n", + "date_information\n", + "Q62894817\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47406390\n", + "date_information\n", + "date_information\n", + "Q3986110\n", + "date_information\n", + "Q19887161\n", + "Q64768142\n", + "date_information\n", + "date_information\n", + "Q66069164\n", + "date_information\n", + "Q28451742\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48838154\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48673974\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15689850\n", + "date_information\n", + "date_information\n", + "Q53775843\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62278093\n", + "Q4332349\n", + "Q65032875\n", + "date_information\n", + "Q1171105\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1061849\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62076011\n", + "date_information\n", + "Q48835820\n", + "date_information\n", + "Q67207071\n", + "date_information\n", + "Q22043860\n", + "date_information\n", + "Q62277630\n", + "date_information\n", + "date_information\n", + "Q65032623\n", + "date_information\n", + "date_information\n", + "Q56542193\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19887159\n", + "Q60286853\n", + "Q51879718\n", + "Q64006322\n", + "Q64006322\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55099304\n", + "date_information\n", + "date_information\n", + "Q64759224\n", + "Q61951885\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17082944\n", + "date_information\n", + "date_information\n", + "Q54858959\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60790967\n", + "date_information\n", + "date_information\n", + "Q77894162\n", + "Q2887372\n", + "Q2887372\n", + "Q1764531\n", + "date_information\n", + "Q16011979\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47500789\n", + "Q59707291\n", + "date_information\n", + "date_information\n", + "Q51930071\n", + "Q63437738\n", + "Q60748770\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q56274620\n", + "Q55634129\n", + "Q1498482\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17080705\n", + "date_information\n", + "Q55818691\n", + "date_information\n", + "date_information\n", + "Q48773837\n", + "Q48773837\n", + "date_information\n", + "Q1219865\n", + "date_information\n", + "Q55605990\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4392552\n", + "Q15633656\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56241156\n", + "date_information\n", + "Q1056692\n", + "Q60738619\n", + "Q47500754\n", + "date_information\n", + "date_information\n", + "Q66309770\n", + "date_information\n", + "Q58411015\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55637246\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768138\n", + "date_information\n", + "Q48811567\n", + "date_information\n", + "Q60743744\n", + "date_information\n", + "Q1052200\n", + "date_information\n", + "Q27\n", + "Q64768541\n", + "date_information\n", + "Q55633790\n", + "Q47383533\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62073524\n", + "date_information\n", + "Q14579154\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q6966375\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15681388\n", + "Q60637508\n", + "Q75261292\n", + "date_information\n", + "Q3567729\n", + "date_information\n", + "date_information\n", + "Q1145625\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55636360\n", + "date_information\n", + "date_information\n", + "Q55609995\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58815093\n", + "date_information\n", + "Q60738390\n", + "Q30609900\n", + "date_information\n", + "Q69741530\n", + "Q69741530\n", + "date_information\n", + "Q30887900\n", + "Q48674017\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48313997\n", + "date_information\n", + "Q64759221\n", + "Q3680696\n", + "Q3680696\n", + "Q1060166\n", + "Q58814954\n", + "date_information\n", + "Q48671641\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4532600\n", + "Q38251511\n", + "Q4079612\n", + "date_information\n", + "date_information\n", + "Q74192895\n", + "Q2371446\n", + "Q2371446\n", + "date_information\n", + "Q64368634\n", + "date_information\n", + "date_information\n", + "Q24041220\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63969269\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12473281\n", + "date_information\n", + "date_information\n", + "Q25610559\n", + "Q19760848\n", + "Q56280077\n", + "Q55614292\n", + "date_information\n", + "date_information\n", + "Q18923795\n", + "Q4325281\n", + "Q4207635\n", + "Q55564530\n", + "Q53721253\n", + "Q65963552\n", + "Q1618265\n", + "date_information\n", + "Q79212167\n", + "date_information\n", + "Q56274942\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56562348\n", + "date_information\n", + "Q27535874\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61450772\n", + "date_information\n", + "Q17558583\n", + "Q57697775\n", + "Q57017331\n", + "Q43528373\n", + "Q57017331\n", + "Q4059278\n", + "date_information\n", + "date_information\n", + "Q55635089\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55983731\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63127116\n", + "Q63127116\n", + "date_information\n", + "date_information\n", + "Q48673895\n", + "date_information\n", + "date_information\n", + "Q64338238\n", + "Q65068630\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4199883\n", + "date_information\n", + "Q64768613\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55762022\n", + "Q74192790\n", + "Q64460046\n", + "date_information\n", + "date_information\n", + "Q60738223\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55635194\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1982853\n", + "Q56692222\n", + "date_information\n", + "Q48733583\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75290417\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12978338\n", + "date_information\n", + "date_information\n", + "Q65924931\n", + "Q56906869\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4131389\n", + "Q55639353\n", + "Q48673975\n", + "Q12164415\n", + "Q26209116\n", + "Q21342433\n", + "date_information\n", + "Q60738009\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56692211\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q3744706\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738839\n", + "Q62277511\n", + "date_information\n", + "Q48700600\n", + "Q12916506\n", + "date_information\n", + "date_information\n", + "Q63183580\n", + "Q2348753\n", + "date_information\n", + "Q18340953\n", + "date_information\n", + "Q21979536\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54958943\n", + "date_information\n", + "Q7312\n", + "Q7312\n", + "date_information\n", + "date_information\n", + "Q12868778\n", + "Q4277663\n", + "date_information\n", + "Q62595549\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60220916\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50649913\n", + "date_information\n", + "Q64157092\n", + "date_information\n", + "date_information\n", + "Q60745425\n", + "date_information\n", + "date_information\n", + "Q4532705\n", + "date_information\n", + "Q21428011\n", + "date_information\n", + "Q56277505\n", + "Q75292970\n", + "date_information\n", + "Q28064471\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25508932\n", + "date_information\n", + "Q28434500\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3874453\n", + "date_information\n", + "date_information\n", + "Q21672950\n", + "date_information\n", + "Q17085747\n", + "date_information\n", + "Q57015954\n", + "date_information\n", + "Q56086939\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56736617\n", + "date_information\n", + "date_information\n", + "Q56452567\n", + "date_information\n", + "Q48817737\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55633925\n", + "date_information\n", + "Q54888784\n", + "Q73537536\n", + "date_information\n", + "Q18518927\n", + "Q18518927\n", + "Q2205928\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q7262511\n", + "date_information\n", + "Q27\n", + "Q2996959\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50649636\n", + "date_information\n", + "Q64768437\n", + "Q56565719\n", + "Q4233137\n", + "Q54622178\n", + "date_information\n", + "Q3998173\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q10923616\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55632806\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63924389\n", + "date_information\n", + "Q2047348\n", + "Q62277294\n", + "Q27\n", + "Q47480069\n", + "date_information\n", + "date_information\n", + "Q3988825\n", + "Q17478844\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3931523\n", + "date_information\n", + "Q62761144\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4349877\n", + "Q63994420\n", + "Q56703414\n", + "date_information\n", + "date_information\n", + "Q61806618\n", + "Q50649803\n", + "Q60737546\n", + "Q2943549\n", + "Q15052901\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12986578\n", + "date_information\n", + "date_information\n", + "Q55590165\n", + "date_information\n", + "Q17107235\n", + "Q17107235\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3520282\n", + "date_information\n", + "Q7295833\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52161645\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66305419\n", + "date_information\n", + "date_information\n", + "Q57373559\n", + "Q59304918\n", + "Q39078731\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q71037823\n", + "date_information\n", + "date_information\n", + "Q2861452\n", + "Q41438112\n", + "date_information\n", + "Q60576656\n", + "date_information\n", + "Q43901166\n", + "date_information\n", + "Q72426584\n", + "Q47894595\n", + "date_information\n", + "date_information\n", + "Q1127911\n", + "date_information\n", + "date_information\n", + "Q19794079\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15689444\n", + "date_information\n", + "date_information\n", + "Q57018894\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30749736\n", + "date_information\n", + "Q4462643\n", + "date_information\n", + "Q63183580\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61450788\n", + "date_information\n", + "date_information\n", + "Q68098810\n", + "Q68098810\n", + "Q77893670\n", + "date_information\n", + "Q65048227\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21103553\n", + "date_information\n", + "date_information\n", + "Q8051197\n", + "date_information\n", + "date_information\n", + "Q55613549\n", + "Q30880265\n", + "date_information\n", + "Q1065386\n", + "date_information\n", + "date_information\n", + "Q56063981\n", + "date_information\n", + "date_information\n", + "Q47500713\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q8849941\n", + "Q63923963\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24729376\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58814779\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16134165\n", + "date_information\n", + "Q59725715\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50389157\n", + "Q5890997\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51874967\n", + "Q48674354\n", + "date_information\n", + "Q10506677\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q43303197\n", + "Q55390545\n", + "date_information\n", + "date_information\n", + "Q60852120\n", + "Q15971720\n", + "Q15971720\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18244319\n", + "Q66764371\n", + "date_information\n", + "date_information\n", + "Q60738328\n", + "Q2940176\n", + "Q64460118\n", + "Q56524185\n", + "date_information\n", + "date_information\n", + "Q48815868\n", + "Q60793315\n", + "Q59725609\n", + "date_information\n", + "date_information\n", + "Q76488917\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66426745\n", + "Q19960535\n", + "Q19960535\n", + "date_information\n", + "date_information\n", + "Q63398330\n", + "Q60738189\n", + "Q54020785\n", + "date_information\n", + "Q12932196\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50821705\n", + "Q55635433\n", + "date_information\n", + "Q76273253\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56291167\n", + "date_information\n", + "Q52145034\n", + "date_information\n", + "date_information\n", + "Q12215667\n", + "date_information\n", + "Q51115846\n", + "date_information\n", + "date_information\n", + "Q16017012\n", + "Q56736603\n", + "date_information\n", + "Q5663147\n", + "Q12009607\n", + "date_information\n", + "Q49169290\n", + "Q3113471\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63450237\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q48802797\n", + "Q52715809\n", + "date_information\n", + "Q15706485\n", + "date_information\n", + "Q19329824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65060125\n", + "Q51754663\n", + "Q51754663\n", + "Q65926251\n", + "Q53850082\n", + "Q54959190\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24483410\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62631119\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19916425\n", + "date_information\n", + "date_information\n", + "Q57305695\n", + "date_information\n", + "date_information\n", + "Q4470976\n", + "date_information\n", + "Q61851090\n", + "Q63119094\n", + "Q60737531\n", + "Q48314237\n", + "date_information\n", + "Q19329824\n", + "Q4293645\n", + "Q46606592\n", + "date_information\n", + "Q63450164\n", + "Q55635245\n", + "Q3858187\n", + "date_information\n", + "date_information\n", + "Q21904895\n", + "date_information\n", + "Q44491908\n", + "Q55394005\n", + "Q48838154\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63936568\n", + "Q59725690\n", + "date_information\n", + "date_information\n", + "Q60738541\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q53866621\n", + "date_information\n", + "date_information\n", + "Q67207178\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48674000\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16029045\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47044251\n", + "Q19907427\n", + "date_information\n", + "Q19848155\n", + "date_information\n", + "Q65090404\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65065339\n", + "date_information\n", + "date_information\n", + "Q3206523\n", + "Q1584799\n", + "Q1584799\n", + "date_information\n", + "date_information\n", + "Q64747722\n", + "date_information\n", + "Q56550947\n", + "Q60742477\n", + "date_information\n", + "date_information\n", + "Q47455556\n", + "Q70216239\n", + "date_information\n", + "Q69629123\n", + "date_information\n", + "Q56756921\n", + "date_information\n", + "Q64174957\n", + "date_information\n", + "Q64577348\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58814954\n", + "date_information\n", + "Q56599922\n", + "date_information\n", + "Q56341064\n", + "Q24324411\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1788143\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25610607\n", + "Q58310059\n", + "date_information\n", + "date_information\n", + "Q60738161\n", + "date_information\n", + "date_information\n", + "Q63565895\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52715530\n", + "Q4193723\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55392250\n", + "Q18527744\n", + "date_information\n", + "Q63700222\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47064364\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19613796\n", + "Q48317293\n", + "Q4470872\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19691592\n", + "date_information\n", + "date_information\n", + "Q66305400\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3987982\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77906352\n", + "date_information\n", + "date_information\n", + "Q59732835\n", + "Q56063528\n", + "Q47063963\n", + "Q57524498\n", + "Q38251747\n", + "date_information\n", + "Q51885874\n", + "date_information\n", + "Q65071983\n", + "date_information\n", + "date_information\n", + "Q15990861\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55638119\n", + "date_information\n", + "Q18527744\n", + "Q18527744\n", + "date_information\n", + "Q65924953\n", + "date_information\n", + "Q3549581\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3209538\n", + "date_information\n", + "Q62277324\n", + "date_information\n", + "Q51834363\n", + "Q24041220\n", + "Q61847417\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q45710096\n", + "date_information\n", + "date_information\n", + "Q48792062\n", + "date_information\n", + "Q60739768\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30766241\n", + "Q64768089\n", + "date_information\n", + "Q56241170\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2940176\n", + "date_information\n", + "Q23565012\n", + "Q64768509\n", + "Q12980738\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56692194\n", + "Q52151197\n", + "date_information\n", + "Q61911477\n", + "Q61911477\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66828793\n", + "date_information\n", + "Q63450143\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17478936\n", + "Q21768346\n", + "date_information\n", + "date_information\n", + "Q3206905\n", + "Q4231430\n", + "date_information\n", + "date_information\n", + "Q62277886\n", + "date_information\n", + "date_information\n", + "Q64768570\n", + "date_information\n", + "Q19329824\n", + "Q66764353\n", + "date_information\n", + "Q61951929\n", + "Q20672424\n", + "date_information\n", + "Q60738569\n", + "date_information\n", + "Q66663995\n", + "Q27\n", + "date_information\n", + "Q4019398\n", + "date_information\n", + "Q47455556\n", + "date_information\n", + "date_information\n", + "Q12913196\n", + "date_information\n", + "date_information\n", + "Q56275856\n", + "Q10368665\n", + "date_information\n", + "Q60575326\n", + "Q43476852\n", + "date_information\n", + "date_information\n", + "Q64768138\n", + "Q1285133\n", + "Q13041037\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60737591\n", + "Q63924459\n", + "date_information\n", + "Q64768308\n", + "date_information\n", + "Q59149494\n", + "Q24728729\n", + "Q65532105\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q64920706\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50649795\n", + "Q65087830\n", + "Q25505774\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q45838757\n", + "date_information\n", + "Q63198193\n", + "date_information\n", + "date_information\n", + "Q61747349\n", + "Q60745215\n", + "Q51836237\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q24336546\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55402704\n", + "Q1049832\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50650117\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20029149\n", + "date_information\n", + "Q2861646\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56703330\n", + "Q27\n", + "Q48671819\n", + "date_information\n", + "date_information\n", + "Q4087902\n", + "Q22043708\n", + "date_information\n", + "Q18398913\n", + "date_information\n", + "Q19847844\n", + "date_information\n", + "date_information\n", + "Q30901563\n", + "date_information\n", + "date_information\n", + "Q55603013\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54868287\n", + "date_information\n", + "Q58815150\n", + "Q48674292\n", + "date_information\n", + "date_information\n", + "Q19908105\n", + "date_information\n", + "Q62018331\n", + "Q65043142\n", + "Q65043142\n", + "Q65122755\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16123155\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12986338\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63398433\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q72219237\n", + "date_information\n", + "date_information\n", + "Q48791886\n", + "Q48791886\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55604143\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q8718447\n", + "Q54554846\n", + "Q54554846\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47500725\n", + "Q4324144\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q63119063\n", + "date_information\n", + "Q28527409\n", + "Q48671922\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60600320\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65076241\n", + "date_information\n", + "Q55603039\n", + "Q28407934\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768067\n", + "date_information\n", + "Q60853109\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4372982\n", + "Q456239\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47090778\n", + "Q66691283\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21338430\n", + "Q57901801\n", + "date_information\n", + "Q48837015\n", + "Q13110027\n", + "Q12989263\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738349\n", + "Q63871479\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q53844848\n", + "Q28839352\n", + "Q28839352\n", + "date_information\n", + "Q6206924\n", + "Q6206924\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17083476\n", + "date_information\n", + "date_information\n", + "Q16017012\n", + "date_information\n", + "Q59725766\n", + "Q54240121\n", + "date_information\n", + "date_information\n", + "Q48700026\n", + "Q48700026\n", + "date_information\n", + "date_information\n", + "Q61861582\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77894281\n", + "date_information\n", + "date_information\n", + "Q16679535\n", + "date_information\n", + "Q16135618\n", + "Q63568445\n", + "date_information\n", + "date_information\n", + "Q60738248\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56006321\n", + "date_information\n", + "date_information\n", + "Q45709616\n", + "date_information\n", + "date_information\n", + "Q60637469\n", + "Q64350350\n", + "Q64350350\n", + "Q55420238\n", + "Q7936508\n", + "Q7936508\n", + "Q48672695\n", + "date_information\n", + "date_information\n", + "Q6145612\n", + "date_information\n", + "Q64759208\n", + "date_information\n", + "date_information\n", + "Q7634567\n", + "date_information\n", + "Q48778903\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3719053\n", + "Q3719053\n", + "Q55624128\n", + "Q54864392\n", + "Q54864392\n", + "date_information\n", + "date_information\n", + "Q56283796\n", + "date_information\n", + "Q28407066\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q25505817\n", + "date_information\n", + "Q21484965\n", + "date_information\n", + "Q12284695\n", + "Q63968536\n", + "Q63968536\n", + "date_information\n", + "date_information\n", + "Q56703337\n", + "date_information\n", + "Q50280810\n", + "Q10542513\n", + "Q65065437\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27835274\n", + "Q55595776\n", + "date_information\n", + "Q54894974\n", + "Q65679602\n", + "date_information\n", + "date_information\n", + "Q58411662\n", + "Q57524491\n", + "Q4124302\n", + "Q12913776\n", + "Q27\n", + "Q60674735\n", + "date_information\n", + "Q28000916\n", + "date_information\n", + "Q61015509\n", + "Q55364056\n", + "Q28406674\n", + "date_information\n", + "date_information\n", + "Q4240259\n", + "Q76488790\n", + "Q65061242\n", + "Q56486741\n", + "Q16028462\n", + "date_information\n", + "Q14942560\n", + "Q3086265\n", + "Q3086265\n", + "date_information\n", + "date_information\n", + "Q24484582\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17081676\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q60738001\n", + "date_information\n", + "date_information\n", + "Q44506948\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4382036\n", + "Q54894996\n", + "date_information\n", + "Q62277511\n", + "Q51818072\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55379558\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3749740\n", + "Q3749740\n", + "Q48845168\n", + "date_information\n", + "date_information\n", + "Q63923963\n", + "date_information\n", + "Q51852910\n", + "Q56278501\n", + "date_information\n", + "date_information\n", + "Q16316783\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q24729316\n", + "date_information\n", + "Q60737786\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q9302591\n", + "Q9302591\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q70220807\n", + "date_information\n", + "Q56651031\n", + "Q75518257\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q51543325\n", + "date_information\n", + "Q12278958\n", + "date_information\n", + "Q21668126\n", + "Q62018325\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q13009165\n", + "date_information\n", + "date_information\n", + "Q65060213\n", + "date_information\n", + "date_information\n", + "Q15615535\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18411469\n", + "date_information\n", + "Q19760872\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48672584\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q56276985\n", + "date_information\n", + "date_information\n", + "Q59693825\n", + "date_information\n", + "date_information\n", + "Q23957301\n", + "date_information\n", + "Q48790272\n", + "Q63371558\n", + "Q51756473\n", + "Q48671393\n", + "date_information\n", + "Q62018336\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61867559\n", + "Q61867559\n", + "Q25203133\n", + "date_information\n", + "date_information\n", + "Q17632127\n", + "Q47485925\n", + "Q47485925\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q4860128\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12988245\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17539165\n", + "Q19329824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64759213\n", + "date_information\n", + "date_information\n", + "Q20712565\n", + "date_information\n", + "Q55637842\n", + "date_information\n", + "date_information\n", + "Q54020958\n", + "date_information\n", + "Q59725898\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58898819\n", + "Q58898846\n", + "Q58898846\n", + "Q63450143\n", + "date_information\n", + "date_information\n", + "Q51879925\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q13010497\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64577372\n", + "date_information\n", + "Q5747\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19461433\n", + "Q58411313\n", + "Q15717576\n", + "date_information\n", + "Q56026007\n", + "Q197331\n", + "Q197331\n", + "Q55635485\n", + "date_information\n", + "Q11012680\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18554460\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q23305110\n", + "date_information\n", + "date_information\n", + "Q52715587\n", + "Q4198706\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50280802\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q9265906\n", + "date_information\n", + "Q48739505\n", + "Q25508381\n", + "Q48550528\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q39060677\n", + "date_information\n", + "Q3810224\n", + "Q24324379\n", + "Q56452535\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65077628\n", + "date_information\n", + "date_information\n", + "Q64355175\n", + "date_information\n", + "Q13478702\n", + "Q55614158\n", + "date_information\n", + "date_information\n", + "Q43463849\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30727118\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12926922\n", + "date_information\n", + "Q60737774\n", + "date_information\n", + "Q1075755\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2972912\n", + "date_information\n", + "date_information\n", + "Q1055275\n", + "date_information\n", + "date_information\n", + "Q6836288\n", + "Q47500790\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q43401863\n", + "date_information\n", + "Q17080556\n", + "date_information\n", + "Q50650337\n", + "date_information\n", + "date_information\n", + "Q65031957\n", + "date_information\n", + "Q60738340\n", + "Q1070459\n", + "Q16989678\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q73538385\n", + "Q55605100\n", + "Q60767033\n", + "Q48673919\n", + "date_information\n", + "Q63936568\n", + "Q17083476\n", + "Q17562822\n", + "Q5990904\n", + "date_information\n", + "Q56816865\n", + "Q56678472\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62073359\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768096\n", + "Q1068903\n", + "Q39075339\n", + "date_information\n", + "date_information\n", + "Q3635242\n", + "Q3635242\n", + "date_information\n", + "Q65066171\n", + "date_information\n", + "Q50650326\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77894866\n", + "date_information\n", + "date_information\n", + "Q55641567\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55946142\n", + "date_information\n", + "Q979254\n", + "Q979254\n", + "date_information\n", + "Q65081771\n", + "date_information\n", + "Q65118726\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47455556\n", + "date_information\n", + "date_information\n", + "Q18910297\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q54817993\n", + "date_information\n", + "Q61759649\n", + "Q65046237\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4349877\n", + "date_information\n", + "Q63994407\n", + "date_information\n", + "date_information\n", + "Q63871479\n", + "Q17082384\n", + "date_information\n", + "Q57246581\n", + "Q80117984\n", + "Q43451770\n", + "date_information\n", + "date_information\n", + "Q4168689\n", + "Q57246581\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q603711\n", + "Q62018336\n", + "Q603711\n", + "Q56277721\n", + "date_information\n", + "date_information\n", + "Q30901011\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56071279\n", + "Q9265906\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1120892\n", + "Q21752640\n", + "Q64768601\n", + "date_information\n", + "Q75279216\n", + "Q75341911\n", + "date_information\n", + "Q51278957\n", + "Q55602989\n", + "Q49758553\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q20463710\n", + "Q25505760\n", + "date_information\n", + "date_information\n", + "Q16016876\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738167\n", + "Q62075999\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q7245965\n", + "date_information\n", + "Q17479256\n", + "date_information\n", + "date_information\n", + "Q51233362\n", + "Q18554460\n", + "Q50623892\n", + "Q27969667\n", + "Q44865478\n", + "Q2287587\n", + "Q1853096\n", + "Q1853096\n", + "Q2827027\n", + "Q27\n", + "Q17081754\n", + "date_information\n", + "date_information\n", + "Q56664002\n", + "Q56664002\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52004168\n", + "date_information\n", + "Q73550176\n", + "Q63119063\n", + "Q64026414\n", + "date_information\n", + "date_information\n", + "Q15689975\n", + "Q42279997\n", + "date_information\n", + "date_information\n", + "Q66098678\n", + "date_information\n", + "Q54868308\n", + "Q55634452\n", + "Q57418840\n", + "Q57418840\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56064452\n", + "Q56199338\n", + "Q16988670\n", + "date_information\n", + "Q17081590\n", + "Q19845028\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75518787\n", + "Q17080124\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q59733907\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19915851\n", + "Q4459376\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64022059\n", + "date_information\n", + "Q37609273\n", + "date_information\n", + "date_information\n", + "Q46625720\n", + "date_information\n", + "Q65677542\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48671397\n", + "date_information\n", + "Q27\n", + "Q3421105\n", + "Q3421105\n", + "Q24907674\n", + "Q65045904\n", + "date_information\n", + "Q61830133\n", + "Q61830133\n", + "date_information\n", + "date_information\n", + "Q16570900\n", + "Q48674765\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12916211\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q38011705\n", + "Q56692217\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64031533\n", + "date_information\n", + "Q27786016\n", + "date_information\n", + "date_information\n", + "Q1336307\n", + "Q1336307\n", + "Q51845845\n", + "date_information\n", + "Q60738417\n", + "Q12987301\n", + "Q3197182\n", + "Q60773660\n", + "date_information\n", + "date_information\n", + "Q28457459\n", + "date_information\n", + "Q11014572\n", + "Q3192937\n", + "Q105959\n", + "date_information\n", + "Q52267697\n", + "Q64768601\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62018324\n", + "date_information\n", + "Q60737570\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q30901566\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19258530\n", + "date_information\n", + "Q53098702\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q16029045\n", + "date_information\n", + "Q21336464\n", + "date_information\n", + "Q16675670\n", + "date_information\n", + "Q30634277\n", + "Q63348188\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60737884\n", + "Q4019811\n", + "date_information\n", + "Q62081531\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12913150\n", + "Q67207072\n", + "Q1054889\n", + "Q2569449\n", + "date_information\n", + "Q55632402\n", + "Q23059522\n", + "Q23059522\n", + "date_information\n", + "Q55615148\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48758493\n", + "date_information\n", + "date_information\n", + "Q56064161\n", + "date_information\n", + "Q55612161\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16304177\n", + "date_information\n", + "date_information\n", + "Q56279396\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q46835085\n", + "Q1563140\n", + "Q1563140\n", + "date_information\n", + "Q7312\n", + "Q7312\n", + "date_information\n", + "Q55403066\n", + "date_information\n", + "Q55633241\n", + "Q24529636\n", + "Q24529636\n", + "date_information\n", + "date_information\n", + "Q53577577\n", + "Q53007522\n", + "Q53007522\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768127\n", + "date_information\n", + "Q25490712\n", + "date_information\n", + "date_information\n", + "Q48771076\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55632310\n", + "date_information\n", + "Q55633790\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48674090\n", + "date_information\n", + "Q20202440\n", + "date_information\n", + "date_information\n", + "Q63565881\n", + "date_information\n", + "date_information\n", + "Q3336535\n", + "date_information\n", + "Q65080880\n", + "date_information\n", + "Q65032636\n", + "Q60575326\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55081824\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65919093\n", + "date_information\n", + "Q55611328\n", + "date_information\n", + "date_information\n", + "Q30901595\n", + "date_information\n", + "Q30901011\n", + "date_information\n", + "Q3680696\n", + "Q3680696\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q60222985\n", + "date_information\n", + "Q55603751\n", + "date_information\n", + "Q4092188\n", + "date_information\n", + "Q56703344\n", + "Q60738712\n", + "date_information\n", + "Q3523359\n", + "date_information\n", + "Q28222734\n", + "date_information\n", + "Q39070959\n", + "Q63859758\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "Q54935610\n", + "date_information\n", + "date_information\n", + "Q1054858\n", + "Q55907451\n", + "Q54864392\n", + "Q54864392\n", + "date_information\n", + "date_information\n", + "Q11157534\n", + "Q62595428\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17084224\n", + "date_information\n", + "date_information\n", + "Q56254328\n", + "date_information\n", + "Q16016291\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q62018331\n", + "Q65043142\n", + "Q65043142\n", + "date_information\n", + "Q58843996\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28434507\n", + "Q4520010\n", + "Q49822556\n", + "date_information\n", + "Q57524505\n", + "date_information\n", + "date_information\n", + "Q46996529\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56279507\n", + "Q75321363\n", + "date_information\n", + "Q77894130\n", + "Q73536578\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56452591\n", + "Q52781693\n", + "Q27\n", + "Q56452620\n", + "date_information\n", + "date_information\n", + "Q12988455\n", + "date_information\n", + "date_information\n", + "Q3989046\n", + "Q55394008\n", + "date_information\n", + "Q64768025\n", + "Q77892368\n", + "Q58815211\n", + "Q47455556\n", + "Q28509222\n", + "Q19907129\n", + "Q4061838\n", + "Q56274092\n", + "Q12156257\n", + "date_information\n", + "date_information\n", + "Q65032618\n", + "date_information\n", + "Q65117469\n", + "date_information\n", + "date_information\n", + "Q19349761\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19907917\n", + "Q24729099\n", + "Q63614867\n", + "date_information\n", + "Q23771092\n", + "Q56878204\n", + "Q60738665\n", + "Q56868186\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q774365\n", + "Q8253991\n", + "Q56276915\n", + "date_information\n", + "Q2035210\n", + "date_information\n", + "Q75256284\n", + "date_information\n", + "Q65075111\n", + "Q56240003\n", + "date_information\n", + "Q48988512\n", + "Q4441321\n", + "Q75336320\n", + "date_information\n", + "date_information\n", + "Q21081425\n", + "Q10542513\n", + "Q61740801\n", + "date_information\n", + "Q31080455\n", + "Q75269403\n", + "Q18708960\n", + "Q50280807\n", + "Q27\n", + "Q11928049\n", + "Q75574432\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3881205\n", + "date_information\n", + "Q59725700\n", + "Q61268438\n", + "date_information\n", + "Q65059746\n", + "date_information\n", + "date_information\n", + "Q18546743\n", + "Q75261325\n", + "Q3213381\n", + "Q55818699\n", + "Q23957665\n", + "Q58041648\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48671900\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q22119552\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q61955952\n", + "Q3731987\n", + "date_information\n", + "date_information\n", + "Q56284238\n", + "Q55163823\n", + "Q55625451\n", + "Q75248108\n", + "Q3175522\n", + "Q64627111\n", + "Q4491432\n", + "Q6560644\n", + "date_information\n", + "Q4076943\n", + "Q1735461\n", + "date_information\n", + "Q16373255\n", + "date_information\n", + "Q60671508\n", + "Q51278866\n", + "date_information\n", + "date_information\n", + "Q75299375\n", + "Q63924441\n", + "Q51026837\n", + "Q51835369\n", + "Q73537979\n", + "date_information\n", + "Q55552441\n", + "date_information\n", + "Q7312\n", + "Q7312\n", + "Q4345832\n", + "Q9312628\n", + "Q55390128\n", + "Q53736577\n", + "date_information\n", + "date_information\n", + "Q75251661\n", + "date_information\n", + "Q27\n", + "Q883568\n", + "date_information\n", + "Q7237205\n", + "Q75388243\n", + "date_information\n", + "date_information\n", + "Q30668079\n", + "date_information\n", + "date_information\n", + "Q63348173\n", + "date_information\n", + "Q19691746\n", + "Q24728138\n", + "date_information\n", + "Q13105441\n", + "date_information\n", + "Q110521\n", + "Q55590165\n", + "Q2515665\n", + "Q2515665\n", + "Q75384531\n", + "Q55607104\n", + "date_information\n", + "date_information\n", + "Q47484410\n", + "date_information\n", + "Q64577679\n", + "Q75304204\n", + "Q21183741\n", + "Q1464130\n", + "date_information\n", + "Q15278116\n", + "Q48671955\n", + "Q331352\n", + "Q331352\n", + "date_information\n", + "Q18234383\n", + "Q67191054\n", + "Q57821032\n", + "Q55635336\n", + "Q27\n", + "Q1147037\n", + "Q4066550\n", + "date_information\n", + "Q21503271\n", + "Q60461858\n", + "Q27\n", + "Q75246969\n", + "Q862775\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3292912\n", + "Q3292912\n", + "date_information\n", + "Q3989046\n", + "date_information\n", + "Q15123159\n", + "Q55610834\n", + "Q1012024\n", + "Q56276996\n", + "Q55603668\n", + "date_information\n", + "Q60737725\n", + "Q56248006\n", + "Q56248006\n", + "Q52063188\n", + "Q21035652\n", + "date_information\n", + "Q60744747\n", + "date_information\n", + "date_information\n", + "Q65060125\n", + "Q26884129\n", + "date_information\n", + "Q44226364\n", + "Q44226364\n", + "date_information\n", + "date_information\n", + "Q55629138\n", + "Q56561387\n", + "date_information\n", + "date_information\n", + "Q51515927\n", + "Q51031099\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4380867\n", + "Q48673975\n", + "date_information\n", + "Q3749740\n", + "Q3749740\n", + "Q73536712\n", + "date_information\n", + "Q2554051\n", + "Q30693497\n", + "Q188605\n", + "Q55625236\n", + "date_information\n", + "date_information\n", + "Q64018069\n", + "Q48674160\n", + "Q48671762\n", + "date_information\n", + "Q67442788\n", + "date_information\n", + "Q19760881\n", + "Q60741350\n", + "date_information\n", + "date_information\n", + "Q3431361\n", + "Q77903089\n", + "date_information\n", + "date_information\n", + "Q1245637\n", + "date_information\n", + "Q13652012\n", + "Q25739739\n", + "Q14933003\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q77895574\n", + "Q50828546\n", + "date_information\n", + "Q75467047\n", + "Q75271130\n", + "Q21183854\n", + "Q63934645\n", + "date_information\n", + "date_information\n", + "Q2461403\n", + "Q1996080\n", + "Q75276810\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q70895877\n", + "date_information\n", + "Q15706008\n", + "Q76074470\n", + "Q48674657\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65559778\n", + "Q3290274\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4382036\n", + "Q63802341\n", + "Q59776759\n", + "Q61483064\n", + "Q3086265\n", + "Q3086265\n", + "date_information\n", + "date_information\n", + "Q1507650\n", + "Q1665346\n", + "Q1665346\n", + "Q1743888\n", + "Q897472\n", + "Q306928\n", + "Q66305414\n", + "Q63259135\n", + "date_information\n", + "Q62736780\n", + "Q60737739\n", + "date_information\n", + "date_information\n", + "Q64996711\n", + "Q64996711\n", + "Q22302425\n", + "Q60737531\n", + "date_information\n", + "Q75268886\n", + "Q22344308\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28064524\n", + "date_information\n", + "Q74460881\n", + "Q54868269\n", + "Q1147037\n", + "Q75485298\n", + "Q1147037\n", + "date_information\n", + "date_information\n", + "Q50825810\n", + "Q63986495\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q47451374\n", + "Q664627\n", + "Q44077104\n", + "Q75399987\n", + "date_information\n", + "Q18511263\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64606091\n", + "Q3336838\n", + "Q3336838\n", + "date_information\n", + "Q76186239\n", + "Q30591334\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q883568\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60750630\n", + "Q60750679\n", + "Q60750679\n", + "Q75872063\n", + "date_information\n", + "Q56648565\n", + "date_information\n", + "Q3429285\n", + "Q63565878\n", + "Q61911606\n", + "Q55641895\n", + "Q75908259\n", + "Q57398740\n", + "date_information\n", + "Q59617591\n", + "Q50821742\n", + "Q56703415\n", + "date_information\n", + "date_information\n", + "Q50649699\n", + "Q75206451\n", + "date_information\n", + "date_information\n", + "Q55614737\n", + "Q56736600\n", + "date_information\n", + "date_information\n", + "Q47969761\n", + "date_information\n", + "date_information\n", + "Q26722753\n", + "date_information\n", + "Q2387732\n", + "Q47293554\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q14327504\n", + "Q14327504\n", + "date_information\n", + "date_information\n", + "Q55659359\n", + "Q60565417\n", + "Q5769416\n", + "date_information\n", + "Q16017234\n", + "Q75278625\n", + "date_information\n", + "Q1070575\n", + "Q3432499\n", + "date_information\n", + "date_information\n", + "Q56275862\n", + "Q43476870\n", + "Q16761120\n", + "Q20085644\n", + "date_information\n", + "Q13677685\n", + "Q13677685\n", + "date_information\n", + "Q75566771\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q4020590\n", + "Q54958420\n", + "date_information\n", + "date_information\n", + "Q9005\n", + "Q65677546\n", + "Q48700092\n", + "Q55634490\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q37553201\n", + "Q3176228\n", + "date_information\n", + "Q3429837\n", + "Q48674540\n", + "Q5675398\n", + "date_information\n", + "date_information\n", + "Q700380\n", + "Q73536842\n", + "Q188605\n", + "date_information\n", + "Q12867338\n", + "Q60738232\n", + "date_information\n", + "date_information\n", + "Q56064161\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q15622114\n", + "Q65051148\n", + "date_information\n", + "date_information\n", + "Q75397190\n", + "date_information\n", + "Q55391785\n", + "date_information\n", + "Q2662932\n", + "date_information\n", + "date_information\n", + "Q73536921\n", + "Q9005\n", + "Q55640205\n", + "Q56678809\n", + "Q50888343\n", + "date_information\n", + "Q48671308\n", + "Q16605834\n", + "Q60737799\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q7074764\n", + "Q52715635\n", + "Q3235756\n", + "date_information\n", + "Q56045021\n", + "Q56225996\n", + "Q58814779\n", + "Q51031282\n", + "date_information\n", + "Q55616601\n", + "date_information\n", + "Q75256284\n", + "date_information\n", + "Q49746290\n", + "Q19906881\n", + "Q4093441\n", + "date_information\n", + "Q376669\n", + "date_information\n", + "Q18912995\n", + "Q75885105\n", + "Q7128861\n", + "Q60791225\n", + "Q7830594\n", + "Q64460079\n", + "Q2865169\n", + "date_information\n", + "Q16832849\n", + "Q4956744\n", + "date_information\n", + "Q25508309\n", + "Q4318960\n", + "Q76344484\n", + "Q3818605\n", + "Q50804871\n", + "date_information\n", + "Q9005\n", + "date_information\n", + "Q25563078\n", + "Q1895150\n", + "Q55258969\n", + "Q60738268\n", + "Q15649511\n", + "Q13811044\n", + "date_information\n", + "Q47063735\n", + "Q4386849\n", + "Q75253659\n", + "Q65090316\n", + "Q18730165\n", + "Q11923812\n", + "date_information\n", + "Q2134675\n", + "date_information\n", + "Q774365\n", + "date_information\n", + "Q56062284\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47498866\n", + "date_information\n", + "Q55762032\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q2150379\n", + "date_information\n", + "Q55635513\n", + "date_information\n", + "date_information\n", + "Q48674507\n", + "Q15629331\n", + "date_information\n", + "Q13811044\n", + "date_information\n", + "Q164518\n", + "date_information\n", + "Q2387732\n", + "date_information\n", + "Q56223448\n", + "Q75250983\n", + "date_information\n", + "Q56886907\n", + "Q114953\n", + "Q42298769\n", + "Q66472629\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768017\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47167138\n", + "Q1870565\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "Q63282851\n", + "Q63924358\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q327356\n", + "Q17462238\n", + "date_information\n", + "date_information\n", + "Q24055383\n", + "date_information\n", + "Q5261141\n", + "Q7312\n", + "Q7312\n", + "date_information\n", + "date_information\n", + "Q12201526\n", + "Q55950925\n", + "date_information\n", + "Q2833398\n", + "date_information\n", + "Q50828038\n", + "date_information\n", + "Q4078100\n", + "Q11402101\n", + "date_information\n", + "date_information\n", + "Q51070711\n", + "Q63951591\n", + "date_information\n", + "Q67171439\n", + "Q66023775\n", + "Q66023775\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50411737\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75874200\n", + "Q59474327\n", + "date_information\n", + "Q2944916\n", + "Q12286002\n", + "Q51844371\n", + "Q51844371\n", + "Q2272984\n", + "date_information\n", + "Q16761120\n", + "date_information\n", + "date_information\n", + "Q1758660\n", + "Q55630320\n", + "Q55907550\n", + "Q48835993\n", + "Q65677544\n", + "date_information\n", + "date_information\n", + "Q57584319\n", + "Q75885107\n", + "date_information\n", + "Q25563078\n", + "Q27\n", + "Q7312\n", + "Q7312\n", + "Q51026837\n", + "Q55590142\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q76250886\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75341381\n", + "date_information\n", + "Q36705119\n", + "Q164518\n", + "Q3603202\n", + "date_information\n", + "date_information\n", + "Q2963134\n", + "Q2891591\n", + "Q2891591\n", + "Q3358612\n", + "Q64414258\n", + "Q52715740\n", + "date_information\n", + "Q56274013\n", + "date_information\n", + "Q3191819\n", + "Q2884747\n", + "Q2884747\n", + "Q3358612\n", + "Q27\n", + "Q55081826\n", + "date_information\n", + "Q59725655\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56290550\n", + "Q56434717\n", + "Q48804187\n", + "Q64010219\n", + "Q64789566\n", + "Q55634490\n", + "date_information\n", + "Q3613953\n", + "Q3613953\n", + "date_information\n", + "Q42322839\n", + "Q5921305\n", + "Q60738295\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60745846\n", + "date_information\n", + "Q8253991\n", + "date_information\n", + "Q51070711\n", + "date_information\n", + "Q48965171\n", + "Q65117495\n", + "Q65117495\n", + "date_information\n", + "date_information\n", + "Q60738142\n", + "Q55950746\n", + "Q15456810\n", + "Q10274143\n", + "date_information\n", + "Q55443439\n", + "Q60310822\n", + "Q60310822\n", + "Q58411513\n", + "Q45710417\n", + "date_information\n", + "date_information\n", + "Q3978256\n", + "Q3978256\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28966251\n", + "date_information\n", + "Q23771092\n", + "Q56308962\n", + "Q55081822\n", + "Q75574432\n", + "Q55070339\n", + "date_information\n", + "date_information\n", + "Q75261282\n", + "date_information\n", + "date_information\n", + "Q50848188\n", + "date_information\n", + "Q74586939\n", + "Q58008825\n", + "Q58008825\n", + "date_information\n", + "Q15854327\n", + "date_information\n", + "date_information\n", + "Q44727126\n", + "Q55391524\n", + "Q75776327\n", + "Q4580282\n", + "date_information\n", + "Q50143448\n", + "Q58411738\n", + "Q56064134\n", + "Q56064134\n", + "date_information\n", + "Q6292594\n", + "Q48965262\n", + "Q18576842\n", + "date_information\n", + "Q44492116\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q34134216\n", + "Q47546716\n", + "Q47546716\n", + "date_information\n", + "Q3660545\n", + "Q56240003\n", + "Q46230954\n", + "Q2137291\n", + "date_information\n", + "Q27\n", + "Q67072745\n", + "date_information\n", + "Q30591104\n", + "Q75560229\n", + "date_information\n", + "Q33170323\n", + "date_information\n", + "date_information\n", + "Q39050904\n", + "date_information\n", + "date_information\n", + "Q73536750\n", + "Q54861541\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55623515\n", + "Q75634523\n", + "Q52159104\n", + "date_information\n", + "Q61466220\n", + "Q12863348\n", + "Q16134448\n", + "Q16134448\n", + "Q74448919\n", + "Q60737559\n", + "Q15278116\n", + "date_information\n", + "date_information\n", + "Q75250845\n", + "date_information\n", + "Q48809669\n", + "date_information\n", + "date_information\n", + "Q61450786\n", + "date_information\n", + "date_information\n", + "Q75243284\n", + "date_information\n", + "date_information\n", + "Q24728388\n", + "Q21993842\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q52715703\n", + "date_information\n", + "date_information\n", + "Q24950627\n", + "Q24950627\n", + "Q5056642\n", + "date_information\n", + "Q21000712\n", + "Q21000712\n", + "date_information\n", + "Q75252209\n", + "Q44077095\n", + "Q64338145\n", + "date_information\n", + "date_information\n", + "Q15278101\n", + "date_information\n", + "date_information\n", + "Q3010170\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47035225\n", + "Q75282866\n", + "date_information\n", + "Q61851107\n", + "Q20792485\n", + "date_information\n", + "Q20891257\n", + "Q15062295\n", + "Q15062295\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q22275256\n", + "Q55599829\n", + "Q47200677\n", + "date_information\n", + "date_information\n", + "Q50168929\n", + "Q12460488\n", + "Q67497715\n", + "Q63183577\n", + "Q65058944\n", + "Q188605\n", + "Q63380908\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17081338\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q19886995\n", + "Q75783716\n", + "Q57779969\n", + "date_information\n", + "Q48815173\n", + "Q2905436\n", + "Q75567379\n", + "date_information\n", + "date_information\n", + "Q57150126\n", + "Q60738612\n", + "date_information\n", + "Q56604162\n", + "Q75388243\n", + "Q61954782\n", + "date_information\n", + "Q63802231\n", + "Q56761838\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q5862523\n", + "Q52151197\n", + "Q2879099\n", + "Q12720302\n", + "date_information\n", + "Q27\n", + "Q60738015\n", + "Q39758953\n", + "Q39758953\n", + "Q55631639\n", + "date_information\n", + "Q48672679\n", + "Q75420141\n", + "Q27\n", + "date_information\n", + "Q60036950\n", + "date_information\n", + "Q5623543\n", + "Q20075808\n", + "date_information\n", + "Q30636729\n", + "date_information\n", + "date_information\n", + "Q56281346\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q327356\n", + "Q15728101\n", + "date_information\n", + "date_information\n", + "Q25218388\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60738495\n", + "Q21707663\n", + "date_information\n", + "Q63924517\n", + "date_information\n", + "Q254200\n", + "Q23955378\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64947919\n", + "date_information\n", + "Q65080698\n", + "Q76276445\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18419557\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q44252992\n", + "Q65049717\n", + "date_information\n", + "Q59656542\n", + "Q40745153\n", + "Q10563008\n", + "date_information\n", + "date_information\n", + "Q75338760\n", + "date_information\n", + "date_information\n", + "Q73792248\n", + "date_information\n", + "Q59240227\n", + "Q47500739\n", + "date_information\n", + "Q27\n", + "Q48671152\n", + "date_information\n", + "Q11904222\n", + "Q60750745\n", + "Q2944916\n", + "date_information\n", + "Q612684\n", + "Q1665346\n", + "Q1665346\n", + "Q1743888\n", + "Q5764567\n", + "date_information\n", + "Q60169300\n", + "Q60169300\n", + "date_information\n", + "Q61890364\n", + "Q65118239\n", + "Q63977741\n", + "Q48770601\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3635242\n", + "Q3635242\n", + "Q59513147\n", + "Q75321315\n", + "Q3471889\n", + "Q25505901\n", + "Q25508765\n", + "date_information\n", + "Q2836351\n", + "Q56225996\n", + "Q327356\n", + "Q2619334\n", + "Q51756971\n", + "Q47793419\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16557779\n", + "Q766543\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q64768480\n", + "Q75280625\n", + "date_information\n", + "date_information\n", + "Q62589387\n", + "Q60467328\n", + "Q60467328\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q60048295\n", + "Q75250730\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65924945\n", + "date_information\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75273751\n", + "Q64018051\n", + "Q51280942\n", + "Q51276514\n", + "Q51276514\n", + "date_information\n", + "Q7334238\n", + "Q7334238\n", + "Q62018324\n", + "date_information\n", + "Q3275248\n", + "Q3670080\n", + "date_information\n", + "Q624927\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q53679499\n", + "Q55229541\n", + "Q38460115\n", + "date_information\n", + "Q20888806\n", + "date_information\n", + "Q60738525\n", + "Q48672781\n", + "Q4223893\n", + "Q24484604\n", + "Q565344\n", + "Q565344\n", + "Q7487312\n", + "date_information\n", + "date_information\n", + "Q24884893\n", + "Q24884895\n", + "Q24884895\n", + "Q24884901\n", + "Q19800856\n", + "date_information\n", + "date_information\n", + "Q4361488\n", + "Q63927162\n", + "Q75270869\n", + "date_information\n", + "Q43476870\n", + "Q43476870\n", + "Q59849898\n", + "Q56604165\n", + "Q19796666\n", + "date_information\n", + "date_information\n", + "Q55217305\n", + "Q49201649\n", + "Q1953994\n", + "Q1953994\n", + "date_information\n", + "Q27\n", + "Q60737714\n", + "date_information\n", + "date_information\n", + "Q22315057\n", + "Q48671481\n", + "date_information\n", + "date_information\n", + "Q16565369\n", + "Q60738678\n", + "date_information\n", + "Q21012765\n", + "date_information\n", + "date_information\n", + "Q55779954\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75304204\n", + "date_information\n", + "Q75272765\n", + "Q55215615\n", + "Q55215630\n", + "Q55215630\n", + "Q27\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1294270\n", + "Q1110722\n", + "Q1110722\n", + "date_information\n", + "Q774365\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56886898\n", + "Q16366420\n", + "Q75301191\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28452116\n", + "date_information\n", + "Q1677908\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75388720\n", + "date_information\n", + "Q48767646\n", + "date_information\n", + "Q17332264\n", + "date_information\n", + "date_information\n", + "Q54958519\n", + "Q24327817\n", + "Q13636928\n", + "Q65491485\n", + "Q65677532\n", + "date_information\n", + "Q75755608\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75249451\n", + "date_information\n", + "Q3731987\n", + "Q63924156\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q22119552\n", + "date_information\n", + "date_information\n", + "Q75873273\n", + "Q12929174\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q56452582\n", + "Q53451557\n", + "date_information\n", + "Q1464130\n", + "Q75404984\n", + "date_information\n", + "date_information\n", + "Q60738682\n", + "Q56562110\n", + "Q54020839\n", + "Q64140897\n", + "Q63370550\n", + "Q59967584\n", + "Q28064471\n", + "Q5495681\n", + "date_information\n", + "date_information\n", + "Q75547305\n", + "date_information\n", + "date_information\n", + "Q75549567\n", + "Q7312\n", + "Q7312\n", + "Q21857711\n", + "Q24728180\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q3823913\n", + "date_information\n", + "date_information\n", + "Q75958041\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q58824947\n", + "Q16075048\n", + "date_information\n", + "Q54868288\n", + "date_information\n", + "Q4349877\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q1013447\n", + "Q5623543\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q75278979\n", + "Q60748770\n", + "Q65042417\n", + "Q50824425\n", + "Q16647517\n", + "Q63698624\n", + "date_information\n", + "Q4587562\n", + "date_information\n", + "date_information\n", + "Q60788703\n", + "Q18234383\n", + "date_information\n", + "Q75257668\n", + "date_information\n", + "date_information\n", + "Q75274930\n", + "Q2287587\n", + "Q1853096\n", + "Q1853096\n", + "date_information\n", + "date_information\n", + "Q60738421\n", + "Q60738489\n", + "Q60738707\n", + "Q55609700\n", + "date_information\n", + "Q1147037\n", + "Q50323445\n", + "date_information\n", + "date_information\n", + "Q47034689\n", + "date_information\n", + "Q48671573\n", + "Q29329953\n", + "Q66486074\n", + "Q66486074\n", + "date_information\n", + "Q56072951\n", + "Q5623543\n", + "Q61704768\n", + "Q14455255\n", + "Q5261141\n", + "Q7312\n", + "Q7312\n", + "Q77899618\n", + "Q58411763\n", + "Q50649795\n", + "Q311536\n", + "date_information\n", + "Q30676547\n", + "Q34211\n", + "Q34211\n", + "date_information\n", + "date_information\n", + "Q50757990\n", + "date_information\n", + "Q55614737\n", + "Q30693497\n", + "Q30693497\n", + "Q51944500\n", + "date_information\n", + "Q47217028\n", + "Q20888806\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q18912661\n", + "Q75320950\n", + "date_information\n", + "Q16859764\n", + "Q16859764\n", + "date_information\n", + "Q55627536\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q57821032\n", + "date_information\n", + "date_information\n", + "Q60737698\n", + "Q75551209\n", + "date_information\n", + "Q56290550\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q17495115\n", + "date_information\n", + "date_information\n", + "Q13018966\n", + "Q63934645\n", + "date_information\n", + "Q63247471\n", + "Q17479036\n", + "Q65552650\n", + "Q75251464\n", + "Q20001576\n", + "Q31080455\n", + "date_information\n", + "date_information\n", + "Q27\n", + "date_information\n", + "Q4413321\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q16028462\n", + "date_information\n", + "Q15724107\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12054532\n", + "Q5298474\n", + "Q5298474\n", + "date_information\n", + "Q65090796\n", + "Q48672540\n", + "Q75270784\n", + "Q50650236\n", + "date_information\n", + "Q75256869\n", + "date_information\n", + "Q55635014\n", + "date_information\n", + "Q55357013\n", + "date_information\n", + "Q3337417\n", + "Q3147770\n", + "date_information\n", + "Q10405126\n", + "Q7312\n", + "Q7312\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q65085211\n", + "Q38460115\n", + "Q16029045\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q66979456\n", + "Q27\n", + "Q27\n", + "Q66023775\n", + "Q66023775\n", + "date_information\n", + "Q15044914\n", + "Q16681889\n", + "Q55762014\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q47064211\n", + "date_information\n", + "Q65089937\n", + "Q28220182\n", + "date_information\n", + "Q60738167\n", + "date_information\n", + "Q47004711\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q55081550\n", + "Q11071898\n", + "date_information\n", + "Q27\n", + "Q67381025\n", + "date_information\n", + "Q1147037\n", + "Q56816886\n", + "date_information\n", + "date_information\n", + "Q3602534\n", + "Q59733171\n", + "date_information\n", + "date_information\n", + "Q20823719\n", + "date_information\n", + "Q7312\n", + "Q7312\n", + "Q4144876\n", + "Q65089554\n", + "date_information\n", + "Q12049291\n", + "date_information\n", + "date_information\n", + "Q12864386\n", + "Q65090283\n", + "Q50378765\n", + "Q23771092\n", + "Q56063974\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q28790662\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q21008003\n", + "date_information\n", + "date_information\n", + "Q56416551\n", + "Q3661337\n", + "date_information\n", + "Q7433528\n", + "date_information\n", + "date_information\n", + "Q24055383\n", + "Q24055383\n", + "Q28365917\n", + "Q50549252\n", + "Q75394075\n", + "Q28357921\n", + "date_information\n", + "Q48965000\n", + "Q50376598\n", + "Q50376598\n", + "Q10873771\n", + "Q1050168\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q12358705\n", + "Q4971452\n", + "Q4971452\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q50821741\n", + "Q16334588\n", + "date_information\n", + "Q16679476\n", + "Q15942114\n", + "Q8857155\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q48674404\n", + "Q75607731\n", + "Q5260256\n", + "date_information\n", + "Q13017966\n", + "date_information\n", + "Q69629123\n", + "Q2082636\n", + "date_information\n", + "Q76022994\n", + "Q27\n", + "Q48740759\n", + "date_information\n", + "Q65085346\n", + "Q39072373\n", + "Q65053202\n", + "date_information\n", + "Q59725862\n", + "date_information\n", + "Q55607923\n", + "date_information\n", + "date_information\n", + "Q61046008\n", + "date_information\n", + "date_information\n", + "Q28403529\n", + "Q7433541\n", + "Q7433541\n", + "date_information\n", + "Q56886875\n", + "date_information\n", + "Q55637523\n", + "Q65053463\n", + "date_information\n", + "Q47969761\n", + "Q11151191\n", + "Q18922345\n", + "Q18922345\n", + "Q75289998\n", + "Q63934649\n", + "date_information\n", + "Q27\n", + "Q46594130\n", + "date_information\n", + "Q60749762\n", + "date_information\n", + "Q55633296\n", + "Q2849237\n", + "Q2849239\n", + "Q2849239\n", + "Q3495180\n", + "Q15878\n", + "Q4069627\n", + "Q18602858\n", + "Q55229541\n", + "date_information\n", + "Q1296004\n", + "Q56092489\n", + "date_information\n", + "date_information\n", + "Q10960429\n", + "Q64212378\n", + "Q4171683\n", + "Q27\n", + "Q75300203\n", + "Q77903030\n", + "Q64009351\n", + "Q894965\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "date_information\n", + "Q20093131\n", + "Q25563078\n", + "Q47538202\n", + "Q3404002\n", + "Q56886868\n", + "date_information\n", + "Q12201526\n", + "Q17194684\n", + "Q56887171\n", + "Q10338720\n", + "Q42279997\n", + "Q21587815\n" + ] + } + ], + "source": [ + "missing_entities = set()\n", + "missing_entity_idx = set()\n", + "for i, row in enumerate(refined_df.itertuples()):\n", + " for trip in row.graph:\n", + " if len(trip) != 3:\n", + " print(trip)\n", + " entities = trip[0], trip[2]\n", + " for entity in entities:\n", + " if entity not in entity_set:\n", + " print(entity)\n", + " missing_entities.add(entity)\n", + " missing_entity_idx.add(i)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6851" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(missing_entity_idx)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(24441, 7)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "refined_df.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idquestion_textanswer_textis_traingraphtypegraph_size
0cdbb82ec0baf11ebab90acde48001122Who is Boraqchin (Wife Of Ögedei)'s father-in-...Genghis KhanTrue[[Q7069334, spouse, Q7519], [Q7519, father, Q7...inference2
1228546780bdd11eba7f7acde48001122What is the date of birth of the director of f...July 21, 1926True[[Q83623, director, Q309214], [Q309214, date o...compositional2
2e5150a5a0bda11eba7f7acde48001122When did the director of film Laughter In Hell...August 25, 1963True[[Q6498212, director, Q529568], [Q529568, date...compositional2
397954d9408b011ebbd84ac1f6bf848b6Do director of film Coolie No. 1 (1995 Film) a...noTrue[[Q2996544, director, Q725970], [Q64768110, di...bridge_comparison4
413cda43c09b311ebbdb0ac1f6bf848b6Are both mountains, Serre Mourene and Monte Ga...noTrue[[Q3382389, country, Q29], [Q3861576, country,...comparison2
........................
24436e6e38bc40bdd11eba7f7acde48001122Where did Coulson Wallop's father study?OxfordFalse[[Q5176173, father, Q6262835], [Q6262835, educ...compositional2
2443735fc19ce0bde11eba7f7acde48001122What is the place of birth of the director of ...IndiaFalse[[Q1637089, director, Q3417929], [Q3417929, pl...compositional2
24438523dfd220bda11eba7f7acde48001122What is the place of birth of the director of ...ReykjavíkFalse[[Q818604, director, Q1766199], [Q1766199, pla...compositional2
24439abebf78408c611ebbd90ac1f6bf848b6Were Massimo Bonanni and Gino Pasqualotto from...yesFalse[[Q460758, country of citizenship, Q38], [Q215...comparison2
24440f5175c840bdd11eba7f7acde48001122Where was the place of burial of Sultan Cem's ...Fatih MosqueFalse[[Q441311, father, Q34503], [Q34503, place of ...compositional2
\n", + "

24441 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " id \\\n", + "0 cdbb82ec0baf11ebab90acde48001122 \n", + "1 228546780bdd11eba7f7acde48001122 \n", + "2 e5150a5a0bda11eba7f7acde48001122 \n", + "3 97954d9408b011ebbd84ac1f6bf848b6 \n", + "4 13cda43c09b311ebbdb0ac1f6bf848b6 \n", + "... ... \n", + "24436 e6e38bc40bdd11eba7f7acde48001122 \n", + "24437 35fc19ce0bde11eba7f7acde48001122 \n", + "24438 523dfd220bda11eba7f7acde48001122 \n", + "24439 abebf78408c611ebbd90ac1f6bf848b6 \n", + "24440 f5175c840bdd11eba7f7acde48001122 \n", + "\n", + " question_text answer_text \\\n", + "0 Who is Boraqchin (Wife Of Ögedei)'s father-in-... Genghis Khan \n", + "1 What is the date of birth of the director of f... July 21, 1926 \n", + "2 When did the director of film Laughter In Hell... August 25, 1963 \n", + "3 Do director of film Coolie No. 1 (1995 Film) a... no \n", + "4 Are both mountains, Serre Mourene and Monte Ga... no \n", + "... ... ... \n", + "24436 Where did Coulson Wallop's father study? Oxford \n", + "24437 What is the place of birth of the director of ... India \n", + "24438 What is the place of birth of the director of ... Reykjavík \n", + "24439 Were Massimo Bonanni and Gino Pasqualotto from... yes \n", + "24440 Where was the place of burial of Sultan Cem's ... Fatih Mosque \n", + "\n", + " is_train graph \\\n", + "0 True [[Q7069334, spouse, Q7519], [Q7519, father, Q7... \n", + "1 True [[Q83623, director, Q309214], [Q309214, date o... \n", + "2 True [[Q6498212, director, Q529568], [Q529568, date... \n", + "3 True [[Q2996544, director, Q725970], [Q64768110, di... \n", + "4 True [[Q3382389, country, Q29], [Q3861576, country,... \n", + "... ... ... \n", + "24436 False [[Q5176173, father, Q6262835], [Q6262835, educ... \n", + "24437 False [[Q1637089, director, Q3417929], [Q3417929, pl... \n", + "24438 False [[Q818604, director, Q1766199], [Q1766199, pla... \n", + "24439 False [[Q460758, country of citizenship, Q38], [Q215... \n", + "24440 False [[Q441311, father, Q34503], [Q34503, place of ... \n", + "\n", + " type graph_size \n", + "0 inference 2 \n", + "1 compositional 2 \n", + "2 compositional 2 \n", + "3 bridge_comparison 4 \n", + "4 comparison 2 \n", + "... ... ... \n", + "24436 compositional 2 \n", + "24437 compositional 2 \n", + "24438 compositional 2 \n", + "24439 comparison 2 \n", + "24440 compositional 2 \n", + "\n", + "[24441 rows x 7 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped.\n", + "refined_df.reset_index(inplace=True, drop=True)\n", + "refined_df" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "is_train\n", + "True 10969\n", + "False 6621\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cleaned_df = refined_df.drop(missing_entity_idx)\n", + "cleaned_df['is_train'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-08-07 17:32:18-- https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\n", + "Resolving www.dropbox.com (www.dropbox.com)... 162.125.13.18, 2620:100:6017:18::a27d:212\n", + "Connecting to www.dropbox.com (www.dropbox.com)|162.125.13.18|:443... connected.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://www.dropbox.com/scl/fi/6bjl5v1fqocokbpeui29t/wikidata5m_all_triplet.txt.gz?rlkey=wnmo016yogiz3nkl8ljk8kcbv [following]\n", + "--2024-08-07 17:32:18-- https://www.dropbox.com/scl/fi/6bjl5v1fqocokbpeui29t/wikidata5m_all_triplet.txt.gz?rlkey=wnmo016yogiz3nkl8ljk8kcbv\n", + "Reusing existing connection to www.dropbox.com:443.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline/CYN8GVBH1Zf45p2xk52RBNz9ng3i0Ft1d1TxWAf6B-X7DoxdfhtcVWojbEFgzQYhMW8HbHQQa9nCpbENjihUPdjoMFziDthGVzFdtNwtA_m4wBsxbvfNcJq3GH-vafjEMNNl-UW48NdeJrm_QsSqsTNg/file# [following]\n", + "--2024-08-07 17:32:18-- https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline/CYN8GVBH1Zf45p2xk52RBNz9ng3i0Ft1d1TxWAf6B-X7DoxdfhtcVWojbEFgzQYhMW8HbHQQa9nCpbENjihUPdjoMFziDthGVzFdtNwtA_m4wBsxbvfNcJq3GH-vafjEMNNl-UW48NdeJrm_QsSqsTNg/file\n", + "Resolving uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com (uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com)... 162.125.13.15, 2620:100:6017:15::a27d:20f\n", + "Connecting to uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com (uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com)|162.125.13.15|:443... connected.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: /cd/0/inline2/CYMsbtoic8kpX2pJ7dhd5-jfQA2se0Jz9bzro30yCIjWHIka93WDTuMt4Ic77oOJSQwfgWsHWSG7D9nyW2Z3q0o0kWM7G6lBWHdJCPB7cuCgdOPLtVzh30BBMccTNM_3ZdVrimhs4DHLYSVuB7daEKfazziXl3Umlhn8PY7-The6RY9SO1ZPj4qBFZjKGbJ9f27UNgwvq5YW8WjTpguwp-1MHBeMBGirO2nVCrCc7lIXaKxNCOmjDnEtWXixKWpIa4k8CWwkZUKc9Torq6dJ2_jsjC6fh7f2RgJlbFPCJaAE_23sXfGZT6FcquKeCl2gDZByvPTrZpMWbYOhceWIG_uYsBUUyMRan1OZcpmsvaabGSm6mhzDqI4n6g4ltrNQGAI/file [following]\n", + "--2024-08-07 17:32:19-- https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline2/CYMsbtoic8kpX2pJ7dhd5-jfQA2se0Jz9bzro30yCIjWHIka93WDTuMt4Ic77oOJSQwfgWsHWSG7D9nyW2Z3q0o0kWM7G6lBWHdJCPB7cuCgdOPLtVzh30BBMccTNM_3ZdVrimhs4DHLYSVuB7daEKfazziXl3Umlhn8PY7-The6RY9SO1ZPj4qBFZjKGbJ9f27UNgwvq5YW8WjTpguwp-1MHBeMBGirO2nVCrCc7lIXaKxNCOmjDnEtWXixKWpIa4k8CWwkZUKc9Torq6dJ2_jsjC6fh7f2RgJlbFPCJaAE_23sXfGZT6FcquKeCl2gDZByvPTrZpMWbYOhceWIG_uYsBUUyMRan1OZcpmsvaabGSm6mhzDqI4n6g4ltrNQGAI/file\n", + "Reusing existing connection to uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com:443.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 177025973 (169M) [application/octet-stream]\n", + "Saving to: ‘wikidata5m_all_triplet.txt.gz’\n", + "\n", + "wikidata5m_all_trip 100%[===================>] 168.82M 96.2MB/s in 1.8s \n", + "\n", + "2024-08-07 17:32:21 (96.2 MB/s) - ‘wikidata5m_all_triplet.txt.gz’ saved [177025973/177025973]\n", + "\n" + ] + } + ], + "source": [ + "# Finally, we do the knowledge graph from wikidata5m\n", + "!wget \"https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\"" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "!gzip -d \"wikidata5m_all_triplet.txt.gz\"" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Q29387131\tP31\tQ5\n", + "\n" + ] + } + ], + "source": [ + "with open('wikidata5m_all_triplet.txt') as f:\n", + " print(f.readline())" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "# Substitute entity codes with their aliases\n", + "# Picking the first alias for each entity (rather arbitrarily)\n", + "alias_map = {}\n", + "rel_alias_map = {}\n", + "for line in open('wikidata5m_entity.txt'):\n", + " parts = line.strip().split('\\t')\n", + " entity_id = parts[0]\n", + " aliases = parts[1:]\n", + " alias_map[entity_id] = aliases[0]\n", + "for line in open('wikidata5m_relation.txt'):\n", + " parts = line.strip().split('\\t')\n", + " relation_id = parts[0]\n", + " relation_name = parts[1]\n", + " rel_alias_map[relation_id] = relation_name" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "21354359it [00:39, 543970.68it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Missing aliases: 15704/64063077\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "full_graph = []\n", + "missing_total = 0\n", + "total = 0\n", + "for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')):\n", + " src, rel, dst = line.strip().split('\\t')\n", + " if src not in alias_map:\n", + " missing_total += 1\n", + " if dst not in alias_map:\n", + " missing_total += 1\n", + " if rel not in rel_alias_map:\n", + " missing_total += 1\n", + " total += 3\n", + " full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)])\n", + "print(f\"Missing aliases: {missing_total}/{total}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[['lalit kumar goel', 'instance of', 'Huamn'],\n", + " ['ugo riccarelli', 'languages spoken, written or signed', 'italian word'],\n", + " ['Road to Paradise (film)', 'director', 'beaudine, william'],\n", + " ['robert van der horst', 'country of citizenship', 'Reino Hulandes'],\n", + " ['wesley pionteck souza', 'member of sports team', 'Santos F.C.'],\n", + " ['Hezekiah Oladipo Davies', 'instance of', 'Huamn'],\n", + " ['kirakossian', 'country of citizenship', 'Hayastani Hanrapetut’yun'],\n", + " ['mountfair, virginia',\n", + " 'located in the administrative territorial entity',\n", + " 'albermarle county, virginia'],\n", + " ['juan pascual azorin', 'place of birth', 'yecla, spain'],\n", + " ['The Twilight Zone/Time Enough at Last',\n", + " 'followed by',\n", + " 'perchance to dream (twilight zone episode)']]" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "full_graph[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "torch.save(full_graph, 'wikimultihopqa_full_graph.pt')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generating Subgraphs" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from profiling_utils import create_remote_backend_from_triplets\n", + "from rag_feature_store import SentenceTransformerFeatureStore\n", + "from rag_graph_store import NeighborSamplingRAGGraphStore\n", + "from torch_geometric.loader import RAGQueryLoader\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", + "from torch_geometric.data import get_features_for_triplets_groups, Data" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={\"batch_size\": 256}).load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 97d15e22a621cd30057d7658fc788b428781f864 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 8 Aug 2024 19:02:04 -0700 Subject: [PATCH 646/752] documentation part 1 --- .../0_Encoding_a_Large_Knowledge_Graph.ipynb | 2143 +++++++++++++++++ examples/llm_plus_gnn/doc/_Introduction.ipynb | 128 + examples/llm_plus_gnn/doc/media/flowchart.svg | 1 + 3 files changed, 2272 insertions(+) create mode 100644 examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb create mode 100644 examples/llm_plus_gnn/doc/_Introduction.ipynb create mode 100644 examples/llm_plus_gnn/doc/media/flowchart.svg diff --git a/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb b/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb new file mode 100644 index 000000000000..5bedae2d8415 --- /dev/null +++ b/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb @@ -0,0 +1,2143 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Encoding a Large Knowledge Graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we are going to walk through how to encode a large knowledge graph for the purposes of Graph RAG. We will provide two examples of how to do so, along with demonstration code." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 1: Building from Already Existing Datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. The same is true for GNN based RAG. Consider the following dataset WebQSP:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset\n", + "\n", + "# Computationally expensive, so running a small sample for now to show off the schema\n", + "# ds = WebQSPDataset('dataset')\n", + "num_questions = 100\n", + "ds = UpdatedWebQSPDataset('small_sample', limit=num_questions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "WebQSP is a dataset that is based off of a subset of the Freebase Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So each entry in the dataset consists of:\n", + "- A question to be answered\n", + "- The answer\n", + "- A knowledge graph subgraph of Freebase that has the context needed to answer the question." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", + " num_rows: 100\n", + "})" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.raw_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'WebQTrn-0',\n", + " 'question': 'what is the name of justin bieber brother',\n", + " 'answer': ['Jaxon Bieber'],\n", + " 'q_entity': ['Justin Bieber'],\n", + " 'a_entity': ['Jaxon Bieber'],\n", + " 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", + " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", + " ['Rudolph Valentino',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", + " ['Record producer',\n", + " 'music.performance_role.regular_performances',\n", + " 'm.012m1vf1'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", + " ['2011 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0yrkr34'],\n", + " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Stratford', 'location.location.containedby', 'Canada'],\n", + " ['Singer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Musicians and Singers'],\n", + " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", + " ['As Long As You Love Me (Audien dubstep mix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kevin Risto', 'people.person.gender', 'Male'],\n", + " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", + " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", + " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", + " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", + " ['American Music Award for Favorite Pop/Rock Album',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc259'],\n", + " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", + " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", + " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Billboard Music Award for Top Streaming Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0njhx1b'],\n", + " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", + " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", + " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", + " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Results Show: Week 7'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['One Less Lonely Girl',\n", + " 'music.album.primary_release',\n", + " 'One Less Lonely Girl'],\n", + " ['Twista', 'people.person.gender', 'Male'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", + " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", + " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", + " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", + " ['Madonna', 'music.artist.genre', 'Pop music'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", + " ['m.0gbm3cg',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", + " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", + " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", + " ['MTV Video Music Award Japan for Best New Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhrwc'],\n", + " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", + " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", + " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['m.0gctwjk',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Series 2, Episode 3'],\n", + " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", + " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", + " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", + " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", + " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Beauty and a Beat',\n", + " 'music.single.versions',\n", + " 'Beauty and a Beat (Wideboys Club Mix)'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Bryan Adams',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male'],\n", + " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", + " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", + " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'music.composition.form', 'Song'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0yrkr34'],\n", + " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", + " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", + " ['Record producer',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Haley James Scott'],\n", + " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", + " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['Kanye West', 'people.person.profession', 'Singer'],\n", + " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", + " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", + " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", + " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", + " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", + " ['RedOne', 'music.artist.genre', 'Rock music'],\n", + " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", + " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", + " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'music.recording.featured_artists',\n", + " 'Big Sean'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", + " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", + " ['Athan Grace', 'people.person.profession', 'Actor'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", + " ['All Around the World (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", + " ['Brandy Norwood',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Whitney Houston'],\n", + " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['MTV Video Music Award for Artist to Watch',\n", + " 'award.award_category.winners',\n", + " 'm.0n1ykxp'],\n", + " ['Caitlin Beadles',\n", + " 'celebrities.celebrity.sexual_relationships',\n", + " 'm.0d33gyj'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audiobot instrumental)'],\n", + " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", + " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'The Mighty Mighty Bosstones'],\n", + " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", + " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", + " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Documentary film'],\n", + " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", + " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Amerie', 'music.artist.genre', 'Pop music'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", + " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep edit)'],\n", + " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Recovery', 'music.recording.song', 'Recovery'],\n", + " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", + " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", + " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", + " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", + " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", + " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", + " ['#thatPower', 'music.recording.song', '#thatPower'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", + " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['2013 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0wjgqck'],\n", + " ['The Island Def Jam Music Group',\n", + " 'organization.organization.parent',\n", + " 'm.04q65lb'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Rusted Root'],\n", + " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['m.0z87d3n',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", + " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", + " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", + " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", + " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", + " ['Tampa', 'location.location.containedby', 'United States of America'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.compositions',\n", + " 'Love Never Felt So Good'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", + " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", + " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", + " ['Estelle', 'people.person.profession', 'Record producer'],\n", + " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", + " ['Chris Jasper', 'people.person.gender', 'Male'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", + " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", + " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", + " ['Snoop Dogg',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['David Nicksay', 'people.person.gender', 'Male'],\n", + " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", + " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Juno Awards of 2014',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0102z0vx'],\n", + " ['As Long As You Love Me (Audiobot remix)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", + " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", + " ['m.0_cyzs_',\n", + " 'celebrities.legal_entanglement.offense',\n", + " 'Driving under the influence'],\n", + " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", + " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", + " ['Mann', 'people.person.gender', 'Male'],\n", + " ['JoJo', 'people.person.gender', 'Female'],\n", + " ['Right Here (featuring Drake)',\n", + " 'music.recording.canonical_version',\n", + " 'Right Here'],\n", + " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", + " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0yrjynf',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", + " ['Pras', 'people.person.profession', 'Record producer'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", + " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", + " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", + " ['Marvin Isley',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['School Gyrls', 'film.film.language', 'English Language'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", + " ['Katy Perry', 'people.person.profession', 'Actor'],\n", + " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", + " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.featured_artists',\n", + " 'Redfoo'],\n", + " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", + " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", + " ['As Long as You Love Me (album version)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Justin Bieber: Just Getting Started',\n", + " 'book.written_work.author',\n", + " 'Justin Bieber'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Marc Maris vs. Ramone'],\n", + " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", + " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", + " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", + " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.releases',\n", + " 'Love Never Felt So Good'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", + " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", + " ['Dance music',\n", + " 'broadcast.genre.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", + " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", + " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", + " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", + " ['Scooter Braun', 'people.person.gender', 'Male'],\n", + " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", + " ['Sir Nolan', 'people.person.gender', 'Male'],\n", + " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", + " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", + " ['Adam Messinger',\n", + " 'music.composer.compositions',\n", + " \"Turn to You (Mother's Day Dedication)\"],\n", + " ['m.0yrktlv',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Male Hottie'],\n", + " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", + " ['Iggy Azalea',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", + " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['m.0gxnnzy',\n", + " 'celebrities.romantic_relationship.relationship_type',\n", + " 'Dated'],\n", + " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", + " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", + " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Britney Spears',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", + " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", + " ['Fergie', 'music.artist.genre', 'Rock music'],\n", + " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", + " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", + " ['Mistletoe', 'music.album.release_type', 'Single'],\n", + " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", + " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.tracks',\n", + " 'Live My Life (Party Rock remix)'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['m.0njw4z2',\n", + " 'award.award_honor.award',\n", + " 'MTV Europe Music Award for Best Male'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", + " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", + " ['m.0gbm3c3',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Miley Cyrus'],\n", + " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", + " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", + " ['JoJo', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Next to You', 'music.album.release_type', 'Single'],\n", + " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", + " ['Willa Ford', 'people.person.languages', 'English Language'],\n", + " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", + " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", + " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", + " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Hot Wired Radio',\n", + " 'broadcast.content.broadcast',\n", + " 'Hot Wired Radio - 128kbps Stream'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", + " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0gbm3d9',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", + " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", + " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", + " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", + " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", + " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", + " ['Justin Timberlake',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Khalil', 'people.person.gender', 'Male'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", + " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", + " ['Selena Gomez',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", + " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", + " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['C1', 'music.artist.genre', 'Hip hop music'],\n", + " ['My Worlds Acoustic',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Album content type'],\n", + " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", + " ['Live My Life', 'music.composition.language', 'English Language'],\n", + " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", + " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", + " ['Baby', 'music.album.releases', 'Baby'],\n", + " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", + " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", + " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", + " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", + " ['Big R Radio - The Hawk',\n", + " 'broadcast.content.artist',\n", + " 'The Black Eyed Peas'],\n", + " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", + " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", + " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", + " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.production_companies',\n", + " 'AEG Live'],\n", + " ['Redfoo', 'people.person.gender', 'Male'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", + " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", + " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", + " ['m.01053qzf',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", + " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", + " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", + " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", + " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", + " ['Ray J', 'people.person.profession', 'Musician'],\n", + " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", + " ['m.0101fv5f',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", + " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", + " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", + " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", + " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Guest host'],\n", + " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", + " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", + " ['Christina Milian', 'people.person.profession', 'Actor'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", + " ['Dance music', 'broadcast.genre.content', '181-party'],\n", + " ['Queen Elizabeth II Diamond Jubilee Medal',\n", + " 'award.award_category.winners',\n", + " 'm.0njwqrb'],\n", + " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", + " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", + " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['CMT Music Award: Collaborative Video of the Year',\n", + " 'award.award_category.winners',\n", + " 'm.0njvs9s'],\n", + " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", + " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", + " ['Chingy', 'people.person.profession', 'Actor'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", + " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", + " ['Justin Bieber',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", + " ['Chingy', 'people.person.gender', 'Male'],\n", + " ['Reed Smoot', 'people.person.gender', 'Male'],\n", + " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", + " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.releases',\n", + " 'As Long As You Love Me (remixes)'],\n", + " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjvlh'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", + " ['Singer', 'common.topic.article', 'm.09l6h'],\n", + " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", + " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0pc670l'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", + " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", + " ['Beyoncé Knowles',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['m.0njhyh_',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Streaming Song (Video)'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", + " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_production_design_by',\n", + " 'Devorah Herbert'],\n", + " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", + " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", + " ['Nick Jonas',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", + " ['Lupe Fiasco',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['Martin Kierszenbaum',\n", + " 'people.person.place_of_birth',\n", + " 'United States of America'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long as You Love Me'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", + " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", + " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", + " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", + " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", + " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0y5tl39',\n", + " 'film.personal_film_appearance.film',\n", + " 'Les Coulisses des Golden Globes'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", + " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z0tgz6'],\n", + " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", + " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", + " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", + " ['Everlast', 'music.artist.label', 'Island Records'],\n", + " ['5th Annual Shorty Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0ywvh8k'],\n", + " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Baby', 'common.topic.notable_types', 'Composition'],\n", + " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['m.0tkqqgg',\n", + " 'award.award_nomination.award',\n", + " 'Juno Award for Pop Album of the Year'],\n", + " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Nasri', 'people.person.profession', 'Singer'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.compositions',\n", + " 'As Long as You Love Me'],\n", + " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", + " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", + " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", + " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", + " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", + " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", + " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", + " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", + " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", + " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", + " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.film',\n", + " 'Zendaya: Behind the Scenes'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", + " ['That Should Be Me', 'music.composition.form', 'Song'],\n", + " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", + " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Justin Bieber'],\n", + " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.song',\n", + " 'Beauty And A Beat'],\n", + " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", + " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Usher', 'music.composer.compositions', 'First Dance'],\n", + " ['m.0n1ykxp',\n", + " 'award.award_honor.award',\n", + " 'MTV Video Music Award for Artist to Watch'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Rockumentary'],\n", + " ['Amerie', 'people.person.gender', 'Female'],\n", + " ['Real Change: Artists for Education',\n", + " 'film.film.personal_appearances',\n", + " 'm.0y5th3r'],\n", + " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", + " ['Beautiful and the Beat',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['PYD', 'common.topic.notable_types', 'Composition'],\n", + " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", + " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", + " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", + " ['iJustine', 'people.person.gender', 'Female'],\n", + " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", + " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", + " ['m.0ng_vkd',\n", + " 'film.personal_film_appearance.film',\n", + " 'A Michael Bublé Christmas'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", + " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", + " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", + " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Vanessa Hudgens',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", + " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", + " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Madonna', 'people.person.profession', 'Record producer'],\n", + " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", + " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", + " ['As Long as You Love Me (acoustic version)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", + " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", + " ['Rob Thomas', 'people.person.gender', 'Male'],\n", + " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", + " ['m.0hvlt03',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", + " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_srv2b'],\n", + " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", + " ['Donna Summer',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0101fszs',\n", + " 'film.personal_film_appearance.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Alanis Morissette',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Official website'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Jenna Andrews'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", + " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Ray J', 'people.person.nationality', 'United States of America'],\n", + " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Nelly Furtado',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['m.0gbm3fj',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", + " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", + " ['As Long As You Love Me (Ferry Corsten club dub)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", + " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", + " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", + " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", + " ['Guest host',\n", + " 'tv.non_character_role.tv_guest_personal_appearances',\n", + " 'm.0v_98y7'],\n", + " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", + " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", + " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", + " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", + " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", + " ['L.A. Reid', 'people.person.gender', 'Male'],\n", + " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['London', 'location.location.containedby', 'Ontario'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_set_decoration_by',\n", + " 'Lia Roldan'],\n", + " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", + " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Change Me', 'music.album.releases', 'Change Me'],\n", + " ['RedOne', 'music.artist.label', 'Island Records'],\n", + " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", + " ['All Around the World',\n", + " 'music.recording.canonical_version',\n", + " 'All Around the World'],\n", + " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0wjhc6c'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", + " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", + " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Lupe Fiasco'],\n", + " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", + " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", + " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", + " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", + " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Beauty and a Beat (Wideboys Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", + " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", + " ['m.0v90skf',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Male Artist'],\n", + " ['Ludacris', 'people.person.profession', 'Actor'],\n", + " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", + " ['Cameo appearance',\n", + " 'tv.special_tv_performance_type.episode_performances',\n", + " 'm.0v1lwt2'],\n", + " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", + " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'One Chance'],\n", + " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", + " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", + " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", + " ['Roller Coaster',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_x4zg3'],\n", + " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", + " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", + " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", + " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", + " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", + " ['m.0sgk_cw',\n", + " 'award.award_honor.award',\n", + " \"Kids' Choice Award for Favorite Song\"],\n", + " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", + " ['MTV Video Music Brazil Award for Best International Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhhqv'],\n", + " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", + " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", + " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", + " ['Britney Spears', 'people.person.profession', 'Singer'],\n", + " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", + " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Rock music',\n", + " 'base.webvideo.internet_video_genre.series',\n", + " 'Biscuithands, The Animated Musical'],\n", + " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", + " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Barry Weiss'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['MTV Europe Music Award for Best Male',\n", + " 'award.award_category.winners',\n", + " 'm.0z1scxk'],\n", + " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", + " ['Will Smith', 'people.person.profession', 'Actor'],\n", + " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", + " ['Will i Am',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Fabian', 'people.person.gender', 'Male'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", + " ['Somebody to Love (remix)',\n", + " 'music.album.primary_release',\n", + " 'Somebody to Love (remix)'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", + " ['Spouse', 'type.property.expected_type', 'Person'],\n", + " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", + " ['Baby', 'music.recording.artist', 'Ludacris'],\n", + " ['Rudolph Valentino',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", + " ['Judy Garland',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Kelly Clarkson',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'm.011h9_22'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", + " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", + " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0vp8rhw',\n", + " 'music.recording_contribution.album',\n", + " 'Beauty and a Beat (Remixes)'],\n", + " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", + " ['Katy Perry: Part of Me',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['m.0jsmvv5',\n", + " 'film.film_regional_release_date.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", + " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", + " ['All Around The World (featuring Ludacris)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", + " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", + " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", + " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", + " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", + " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep mix)'],\n", + " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Fergie', 'people.person.profession', 'Actor'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", + " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", + " ['Zac Efron', 'people.person.gender', 'Male'],\n", + " ['P!nk', 'music.artist.genre', 'Rock music'],\n", + " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", + " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", + " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", + " ['Under the Mistletoe',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Slick Rick'],\n", + " ['Amerie', 'music.artist.genre', 'Rock music'],\n", + " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0pbzq13',\n", + " 'film.performance.special_performance_type',\n", + " 'Cameo appearance'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", + " ['Height', 'type.property.unit', 'Meter'],\n", + " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", + " ['Ray J',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Boyfriend (acoustic version)',\n", + " 'music.recording.canonical_version',\n", + " 'Boyfriend'],\n", + " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Believe Tour',\n", + " 'music.concert_tour.album_or_release_supporting',\n", + " 'Believe'],\n", + " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", + " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", + " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", + " ['Frank Ocean',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['As Long As You Love Me (Audiobot instrumental)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Usher', 'people.person.nationality', 'United States of America'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", + " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", + " ['m.0gbm3b7',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Sheryl Crow',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['All That Matters',\n", + " 'music.composition.composer',\n", + " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Nasri', 'music.artist.genre', 'Reggae'],\n", + " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", + " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", + " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", + " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", + " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", + " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", + " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long As You Love Me'],\n", + " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", + " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", + " ['Whitney Houston', 'people.person.profession', 'Model'],\n", + " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['As Long as You Love Me',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Disney Parks Christmas Day Parade',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['Ray J', 'people.person.profession', 'Artist'],\n", + " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", + " ['American Music Award for Favorite Pop/Rock Male Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc0sf'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", + " ['The Notorious B.I.G.',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", + " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", + " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", + " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", + " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Haley James Scott',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Record producer'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", + " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", + " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['My Worlds: The Collection',\n", + " 'music.album.album_content_type',\n", + " 'Compilation album'],\n", + " ['Lolly', 'music.album.releases', 'Lolly'],\n", + " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", + " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", + " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", + " ['Ja Rule', 'people.person.profession', 'Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", + " ['Die in Your Arms',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z85qxq'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", + " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", + " ['Kuk Harrell',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0101ft5f'],\n", + " ['Somebody to Love (J Stax remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'film.film.executive_produced_by',\n", + " 'Allison Kaye Scarinzi'],\n", + " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", + " ['Nasri', 'music.artist.genre', 'Pop music'],\n", + " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", + " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", + " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.05sp405',\n", + " 'organization.organization_relationship.child',\n", + " 'Island Records'],\n", + " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", + " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", + " ['John Mamann',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Teen Choice Award for Choice Summer Music Star: Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjynf'],\n", + " ['Juicy J', 'people.person.profession', 'Actor'],\n", + " ['m.0yqflrk',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'celebritynetworth.com'],\n", + " ['Miley Cyrus',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", + " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['m.04q65lb',\n", + " 'organization.organization_relationship.child',\n", + " 'The Island Def Jam Music Group'],\n", + " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", + " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", + " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['My World', 'music.album.artist', 'Justin Bieber'],\n", + " ['Don Henley', 'people.person.gender', 'Male'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Musician', 'people.profession.specializations', 'Singer'],\n", + " ['Die in Your Arms',\n", + " 'music.recording.canonical_version',\n", + " 'Die in Your Arms'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['m.0njvs9s',\n", + " 'award.award_honor.award',\n", + " 'CMT Music Award: Collaborative Video of the Year'],\n", + " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber',\n", + " 'music.artist.album',\n", + " 'Turn to You (Mother’s Day Dedication)'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", + " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", + " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Terence Dudley', 'people.person.gender', 'Male'],\n", + " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Mr. Sosa & The Yayo'],\n", + " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", + " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", + " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0101fvbf',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['m.0yrhrwc',\n", + " 'award.award_honor.ceremony',\n", + " '2011 MTV Video Music Aid Japan'],\n", + " ['MTV Europe Music Award for Best North American Act',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhmll'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['m.0101ft1d',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", + " ['Ciara', 'people.person.gender', 'Female'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", + " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", + " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0jzrrqs',\n", + " 'location.mailing_address.country',\n", + " 'United States of America'],\n", + " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", + " ['Big R Radio - Top 40 Hits',\n", + " 'broadcast.content.artist',\n", + " 'Natasha Bedingfield'],\n", + " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ...],\n", + " 'choices': []}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.raw_dataset[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Although this dataset can be trained on as-is, a couple problems emerge from doing so:\n", + "1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond to the algorithm that was used to generate the dataset subgraphs.\n", + "2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a result, it makes sense in this scenario to be able to encode all the entries into a large knowledge graph, so that duplicate nodes and edges can be avoided, and so that alternative retrieval algorithms can be tried. We can do this with the LargeGraphIndexer class:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "import time\n", + "import torch\n", + "import tqdm\n", + "from itertools import chain\n", + "import networkx as nx" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkr34'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'), ('As Long As You Love Me (Ferry Corsten radio)', 'common.topic.notable_types', 'Musical Recording'), ('Toby Gad', 'music.artist.genre', 'Rhythm and blues'), ('Stratford', 'location.location.containedby', 'Canada'), ('Singer', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('Enrique Iglesias', 'people.person.profession', 'Singer'), ('Beauty and a Beat (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'), ('50 Cent', 'people.person.profession', 'Film Producer'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Kevin Risto', 'people.person.gender', 'Male'), ('Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'), ('Shaggy', 'broadcast.artist.content', 'HitzRadio.com'), ('Mary J. Blige', 'people.person.profession', 'Record producer'), ('Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'), ('Paul Anka', 'common.topic.notable_types', 'Musical Artist'), ('m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'), ('Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'), ('m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'), ('1Club.FM: V101', 'broadcast.content.artist', 'The Roots'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('American Music Award for Favorite Pop/Rock Album', 'award.award_category.winners', 'm.0ndc259'), ('A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'), ('Ontario', 'location.administrative_division.country', 'Canada'), ('1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'), ('Music Producer', 'common.topic.subject_of', 'POPPMusic.net'), ('Billboard Music Award for Top Streaming Artist', 'award.award_category.winners', 'm.0njhx1b'), ('Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"), ('Heartbreaker', 'music.composition.recordings', 'Heartbreaker'), ('Brandy Norwood', 'people.person.profession', 'Singer'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'), ('Justin Bieber', 'music.artist.album', 'All Bad'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('m.0v_729v', 'tv.tv_guest_personal_appearance.episode', 'Results Show: Week 7'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'), ('One Less Lonely Girl', 'music.album.primary_release', 'One Less Lonely Girl'), ('Twista', 'people.person.gender', 'Male'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'), ('Ciara', 'broadcast.artist.content', 'FLOW 103'), ('Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Somebody To Love', 'music.recording.artist', 'Justin Bieber'), ('Toby Gad', 'music.artist.genre', 'Rock music'), ('Madonna', 'music.artist.genre', 'Pop music'), ('Selena Gomez', 'music.artist.genre', 'Europop'), ('m.0gbm3cg', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Baby', 'music.recording.canonical_version', 'Baby'), ('Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Dr. Dre', 'music.artist.genre', 'Rap music'), ('MTV Video Music Award Japan for Best New Artist', 'award.award_category.winners', 'm.0yrhrwc'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Hip hop music', 'broadcast.genre.content', 'FLOW 103'), ('Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('m.0gctwjk', 'tv.tv_guest_role.episodes_appeared_in', 'Series 2, Episode 3'), ('Enrique Iglesias', 'music.artist.genre', 'Dance-pop'), ('Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'), ('FLOW 103', 'broadcast.content.genre', 'Hip hop music'), ('Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Selena Gomez', 'people.person.profession', 'Dancer'), ('Little Bird', 'music.recording.tracks', 'm.0v2hrym'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'), ('Never Say Never', 'common.topic.notable_types', 'Musical Recording'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Club Mix)'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Madonna', 'people.person.profession', 'Singer-songwriter'), ('Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'), ('Terence Dudley', 'music.artist.genre', 'Reggae'), ('Kylie Minogue', 'people.person.profession', 'Actor'), ('Adrienne Bailon', 'music.artist.genre', 'Pop music'), ('Katy Perry', 'music.artist.genre', 'Electronic music'), ('Dany Brillant', 'people.person.gender', 'Male'), ('Martin Kierszenbaum', 'people.person.gender', 'Male'), ('Anastacia', 'people.person.nationality', 'United States of America'), ('Amerie', 'music.artist.label', 'The Island Def Jam Music Group'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Children'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Somebody to Love', 'music.composition.form', 'Song'), ('Teen Choice Award for Choice Twitter Personality', 'award.award_category.winners', 'm.0yrkr34'), ('Chef Tone', 'people.person.place_of_birth', 'Chicago'), ('Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Haley James Scott'), ('Colbie Caillat', 'music.artist.genre', 'Pop music'), ('C1', 'music.artist.genre', 'Contemporary R&B'), ('Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'), ('Kanye West', 'people.person.profession', 'Singer'), ('Pop music', 'common.topic.subject_of', 'Stephen Melton'), ('radioIO Todays POP', 'broadcast.content.producer', 'Radioio'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'), ('Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'), ('Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'), ('Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'), ('Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'), ('Adam Messinger', 'music.composer.compositions', 'Mistletoe'), ('Live My Life', 'music.album.compositions', 'Live My Life'), ('RedOne', 'music.artist.genre', 'Rock music'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'), ('Terius Nash', 'music.artist.genre', 'Rhythm and blues'), ('Little Bird', 'common.topic.notable_types', 'Musical Recording'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.featured_artists', 'Big Sean'), ('Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Terius Nash', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'), ('Athan Grace', 'people.person.profession', 'Actor'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'), ('All Around the World (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Bad Day', 'music.composition.composer', 'Marvin Isley'), ('Brandy Norwood', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('MTV Video Music Award for Artist to Watch', 'award.award_category.winners', 'm.0n1ykxp'), ('Caitlin Beadles', 'celebrities.celebrity.sexual_relationships', 'm.0d33gyj'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot instrumental)'), ('Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'), ('School Boy Records', 'music.record_label.artist', 'Scooter Braun'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Mighty Mighty Bosstones'), ('m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'), ('Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'), ('Lolly', 'music.composition.composer', 'Juicy J'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Documentary film'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Amerie', 'music.artist.genre', 'Pop music'), ('1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'), ('Rodney Jerkins', 'music.artist.genre', 'Synthpop'), ('WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien dubstep edit)'), ('Will Smith', 'broadcast.artist.content', 'Sunshine Radio'), ('Recovery', 'music.recording.song', 'Recovery'), ('Justin Timberlake', 'music.artist.genre', 'Electronic music'), ('Mannie Fresh', 'people.person.nationality', 'United States of America'), ('m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Benny Blanco', 'common.topic.notable_types', 'Record Producer'), ('Leif Garrett', 'music.artist.genre', 'Rock music'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'), ('First Dance', 'music.recording.artist', 'Justin Bieber'), ('#thatPower', 'music.recording.song', '#thatPower'), ('Children', 'rdf-schema#range', 'Person'), ('Beautiful', 'common.topic.notable_for', 'g.1256glpl9'), ('Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjgqck'), ('The Island Def Jam Music Group', 'organization.organization.parent', 'm.04q65lb'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Rusted Root'), ('radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'), ('m.0z87d3n', 'award.award_honor.award', 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'), ('Shaffer Smith', 'music.artist.genre', 'Dance music'), ('Live My Life', 'music.composition.composer', 'John Mamann'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'), ('m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Trick Daddy', 'broadcast.artist.content', 'PowerHitz'), ('1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'), ('Tampa', 'location.location.containedby', 'United States of America'), ('Love Never Felt So Good', 'music.album.compositions', 'Love Never Felt So Good'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.artist', 'Justin Bieber'), ('Nelly', 'music.artist.genre', 'Rhythm and blues'), ('Marvin Isley', 'music.composer.compositions', 'Bad Day'), ('Somebody to Love', 'common.topic.notable_types', 'Composition'), ('Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'), ('Snoop Dogg', 'people.person.gender', 'Male'), ('DMX', 'broadcast.artist.content', '.977 The Hits Channel'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'), ('Estelle', 'people.person.profession', 'Record producer'), ('m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('PowerHitz', 'broadcast.content.genre', 'Hip hop music'), ('Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('50 Cent', 'people.person.nationality', 'United States of America'), ('Chris Jasper', 'people.person.gender', 'Male'), ('Sir Nolan', 'music.artist.genre', 'Pop music'), ('Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('David Nicksay', 'people.person.gender', 'Male'), ('Justin Bieber', 'people.person.profession', 'Record producer'), ('Everlast', 'people.person.profession', 'Singer-songwriter'), ('Juno Awards of 2014', 'award.award_ceremony.awards_presented', 'm.0102z0vx'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.song', 'As Long as You Love Me'), ('#thatPower', 'music.composition.composer', 'Will i Am'), ('m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'), ('m.0_cyzs_', 'celebrities.legal_entanglement.offense', 'Driving under the influence'), ('LeAnn Rimes', 'people.person.profession', 'Actor'), ('KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'), ('1Club.FM: Power', 'broadcast.content.artist', 'Usher'), ('Mann', 'people.person.gender', 'Male'), ('JoJo', 'people.person.gender', 'Female'), ('Right Here (featuring Drake)', 'music.recording.canonical_version', 'Right Here'), ('Mason Levy', 'music.composer.compositions', 'Boyfriend'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('m.0yrjynf', 'award.award_honor.award', 'Teen Choice Award for Choice Summer Music Star: Male'), ('Pras', 'people.person.profession', 'Record producer'), ('1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'), ('Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Raekwon', 'broadcast.artist.content', 'Smoothbeats'), ('Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'), ('Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'), ('My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('School Gyrls', 'film.film.language', 'English Language'), ('Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'), ('Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'), ('Katy Perry', 'people.person.profession', 'Actor'), ('As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'), ('Ronald Isley', 'people.person.profession', 'Actor'), ('Live My Life (Party Rock remix)', 'music.recording.featured_artists', 'Redfoo'), ('HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'), ('Jaxon Bieber', 'people.person.nationality', 'Canada'), ('As Long as You Love Me (album version)', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber: Just Getting Started', 'book.written_work.author', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Marc Maris vs. Ramone'), ('Gwen Stefani', 'people.person.profession', 'Musician'), ('m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'), ('Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'), ('Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'), ('Love Never Felt So Good', 'music.album.releases', 'Love Never Felt So Good'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Sir Mix-a-Lot', 'people.person.profession', 'Actor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'), ('Dance music', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('1Club.FM: V101', 'broadcast.content.location', 'Chicago'), ('Terius Nash', 'people.person.profession', 'Record producer'), ('Terence Dudley', 'people.person.profession', 'Record producer'), ('Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'), ('Baby', 'common.topic.notable_types', 'Award-Winning Work'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Scooter Braun', 'people.person.gender', 'Male'), ('Mistletoe', 'music.album.artist', 'Justin Bieber'), ('Sir Nolan', 'people.person.gender', 'Male'), ('My Worlds: The Collection', 'music.album.genre', 'Teen pop'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'), ('Synthpop', 'music.genre.parent_genre', 'K-pop'), ('Adam Messinger', 'music.composer.compositions', \"Turn to You (Mother's Day Dedication)\"), ('m.0yrktlv', 'award.award_honor.award', 'Teen Choice Award for Choice Male Hottie'), ('Kanye West', 'people.person.nationality', 'United States of America'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'), ('Juicy J', 'freebase.valuenotation.has_value', 'Parents'), ('JellyRadio.com', 'broadcast.content.artist', 'DMX'), ('HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'), ('m.0gxnnzy', 'celebrities.romantic_relationship.relationship_type', 'Dated'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'), ('radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'), ('m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Britney Spears', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Frank Ocean', 'music.artist.genre', 'Rhythm and blues'), ('Ludacris', 'music.artist.contribution', 'm.0vp800w'), ('Singer', 'common.topic.subject_of', 'Justin Bieber'), ('Fergie', 'music.artist.genre', 'Rock music'), ('Gas Pedal', 'common.topic.notable_types', 'Musical Recording'), ('Toby Gad', 'people.person.profession', 'Record producer'), ('All Around The World', 'music.composition.composer', 'Justin Bieber'), ('Mistletoe', 'music.album.release_type', 'Single'), ('Kid Cudi', 'people.person.profession', 'Film Producer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'), ('Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.artist', 'Justin Bieber'), ('m.0njw4z2', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), (\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'), ('Snoop Dogg', 'people.person.profession', 'Record producer'), ('Savan Kotecha', 'music.artist.genre', 'Dance-pop'), ('m.0gbm3c3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Rodney Jerkins', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Miley Cyrus'), ('Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), (\"Destiny's Child\", 'music.artist.genre', 'Pop music'), ('United States of America', 'base.biblioness.bibs_topic.is_really', 'United States of America'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.09xx941', 'common.webpage.topic', 'Teen idol'), ('Christina Milian', 'people.person.profession', 'Record producer'), ('JoJo', 'people.person.nationality', 'United States of America'), ('Kylie Minogue', 'music.artist.genre', 'Electronic dance music'), ('Next to You', 'music.album.release_type', 'Single'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('Willa Ford', 'people.person.languages', 'English Language'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All That Matters', 'music.composition.composer', 'Andre Harris'), ('Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'), ('Paul Anka', 'music.artist.genre', 'Pop music'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'), ('Caitlin Beadles', 'people.person.nationality', 'Canada'), ('m.0z8s_wn', 'award.award_honor.honored_for', 'My World'), ('Favorite Girl', 'common.topic.notable_types', 'Musical Album'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'), ('Avery', 'common.topic.notable_types', 'Musical Artist'), ('m.0gbm3d9', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Beyoncé Knowles', 'people.person.profession', 'Actor'), ('m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Tupac Shakur', 'people.person.profession', 'Actor'), ('Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'), ('Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'), ('Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Frank Ocean', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'music.composer.compositions', 'Baby'), ('Chance the Rapper', 'music.artist.genre', 'Hip hop music'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Khalil', 'people.person.gender', 'Male'), ('#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Selena Gomez', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'), ('Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'), ('m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'), ('Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'), ('Anastacia', 'music.artist.genre', 'Contemporary R&B'), ('C1', 'music.artist.genre', 'Hip hop music'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Album content type'), ('m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'), ('Live My Life', 'music.composition.language', 'English Language'), ('Vocals', 'music.instrument.instrumentalists', 'Aaliyah'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'), ('Baby', 'music.album.releases', 'Baby'), ('A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'), ('Right Here', 'music.recording.canonical_version', 'Right Here'), ('Justin Bieber', 'people.person.profession', 'Musician'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bigger', 'music.composition.composer', 'Waynne Nugent'), ('Home to Mama', 'music.composition.composer', 'Cody Simpson'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Thought Of You', 'music.composition.composer', 'Justin Bieber'), ('The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'), ('Singer', 'people.profession.specializations', 'Prima donna'), ('Alanis Morissette', 'people.person.profession', 'Record producer'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'), ('Record producer', 'common.topic.notable_for', 'g.1258k9617'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'), ('Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'), ('Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'AEG Live'), ('Redfoo', 'people.person.gender', 'Male'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'), ('WildFMRadio.com', 'broadcast.content.artist', '50 Cent'), ('Ronald Isley', 'music.artist.genre', 'Quiet Storm'), ('Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Right Here', 'music.album.featured_artists', 'Drake'), ('m.01053qzf', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Guglielmo Scilla', 'common.topic.notable_types', 'Person'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'), ('Jordan Pruitt', 'music.artist.genre', 'Pop music'), ('Mason Levy', 'music.artist.genre', 'Rhythm and blues'), ('Thought of You', 'common.topic.notable_types', 'Canonical Version'), ('Whitney Houston', 'people.person.profession', 'Record producer'), ('m.07lkzw7', 'common.webpage.category', 'Official Website'), ('Ray J', 'people.person.profession', 'Musician'), ('m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Enrique Iglesias', 'people.person.gender', 'Male'), ('m.0101fv5f', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('HitzRadio.com', 'broadcast.content.artist', 'Nelly'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('Selena Gomez', 'music.artist.genre', 'Teen pop'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'), ('Love Never Felt So Good', 'music.album.genre', 'Disco'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'), ('m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_729v', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'), ('Ja Rule', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('Christina Milian', 'people.person.profession', 'Actor'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Dance music', 'broadcast.genre.content', '181-party'), ('Queen Elizabeth II Diamond Jubilee Medal', 'award.award_category.winners', 'm.0njwqrb'), ('Sean Kingston', 'people.person.profession', 'Singer'), ('DMX', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'), ('CMT Music Award: Collaborative Video of the Year', 'award.award_category.winners', 'm.0njvs9s'), ('m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'), ('One Time', 'common.topic.notable_types', 'Musical Album'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'), ('Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'), ('Katy Perry', 'music.artist.genre', 'Disco'), ('Chingy', 'people.person.profession', 'Actor'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'), ('Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'), ('Rihanna', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Contemporary R&B', 'common.topic.notable_types', 'Musical genre'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'City High'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'), ('Chingy', 'people.person.gender', 'Male'), ('Reed Smoot', 'people.person.gender', 'Male'), (\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'), ('Teyana', 'freebase.valuenotation.has_value', 'Parents'), ('Next to You', 'music.recording.song', 'Next to You'), ('All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('As Long as You Love Me', 'music.album.releases', 'As Long As You Love Me (remixes)'), ('Teen Choice Award for Choice Music: Breakout Artist - Male', 'award.award_category.winners', 'm.0yrjvlh'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'), ('Singer', 'common.topic.article', 'm.09l6h'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'), ('Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber: Never Say Never', 'award.award_winning_work.awards_won', 'm.0pc670l'), ('1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'), ('Beauty And A Beat', 'music.composition.form', 'Song'), ('Britney Spears', 'music.artist.genre', 'Electronic dance music'), ('HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('m.0njhyh_', 'award.award_honor.award', 'Billboard Music Award for Top Streaming Song (Video)'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'), ('Ludacris', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Never Say Never', 'film.film.film_production_design_by', 'Devorah Herbert'), ('Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Drake', 'music.artist.genre', 'Rhythm and blues'), ('Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'), ('Lupe Fiasco', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Martin Kierszenbaum', 'people.person.place_of_birth', 'United States of America'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'), ('m.0d_hbgr', 'common.webpage.category', 'Lyrics'), ('Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'), ('Beautiful', 'music.composition.lyricist', 'Toby Gad'), ('Redfoo', 'music.artist.genre', 'Electronic dance music'), ('1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'), ('K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'), ('K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'), ('Stephen Melton', 'music.group_member.instruments_played', 'Vocals'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Contemporary R&B', 'broadcast.genre.content', '181-thebox'), ('Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'), ('m.0y5tl39', 'film.personal_film_appearance.film', 'Les Coulisses des Golden Globes'), ('Teen idol', 'common.topic.webpage', 'm.09y89l2'), ('m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kevin Risto', 'people.person.profession', 'Musician'), ('Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z0tgz6'), ('Justin Bieber', 'music.artist.label', 'Island Records'), ('Ernie Isley', 'people.person.nationality', 'United States of America'), ('Kylie Minogue', 'people.person.profession', 'Film Producer'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'), ('Everlast', 'music.artist.label', 'Island Records'), ('5th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0ywvh8k'), ('Chance the Rapper', 'music.featured_artist.albums', 'Confident'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'), ('Baby', 'common.topic.notable_types', 'Composition'), ('Fabian', 'base.icons.icon.icon_genre', 'Teen idol'), ('Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0tkqqgg', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Person', 'type.type.properties', 'Parents'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nasri', 'people.person.profession', 'Singer'), ('Lady Gaga', 'music.artist.genre', 'Contemporary R&B'), ('Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('As Long as You Love Me', 'music.album.compositions', 'As Long as You Love Me'), ('Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'), ('The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'), ('Bigger', 'music.composition.composer', 'Frank Ocean'), ('Bigger', 'music.composition.recordings', 'Bigger'), ('Canadian', 'common.topic.notable_types', 'Ethnicity'), ('As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'), ('Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chef Tone', 'people.person.nationality', 'United States of America'), ('Whitney Houston', 'music.artist.genre', 'Dance music'), ('My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'), ('Avery', 'music.artist.label', 'The Island Def Jam Music Group'), ('Change Me', 'music.album.primary_release', 'Change Me'), ('Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('m.0w3gbtv', 'film.personal_film_appearance.film', 'Zendaya: Behind the Scenes'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'), ('That Should Be Me', 'music.composition.form', 'Song'), ('Never Say Never', 'music.album.compositions', 'Never Say Never'), ('m.09wsj7g', 'common.webpage.topic', 'Teen idol'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Justin Bieber'), ('#thatPOWER', 'music.album.releases', '#thatPOWER'), ('Ashley Tisdale', 'people.person.profession', 'Actor'), ('Sir Nolan', 'music.artist.genre', 'Rock music'), ('Beauty and a Beat (acoustic version)', 'music.recording.song', 'Beauty And A Beat'), ('Ellen DeGeneres', 'people.person.nationality', 'United States of America'), ('Sia Furler', 'people.person.profession', 'Singer-songwriter'), ('Usher', 'music.composer.compositions', 'First Dance'), ('m.0n1ykxp', 'award.award_honor.award', 'MTV Video Music Award for Artist to Watch'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Rockumentary'), ('Amerie', 'people.person.gender', 'Female'), ('Real Change: Artists for Education', 'film.film.personal_appearances', 'm.0y5th3r'), ('Mistletoe', 'music.album.primary_release', 'Mistletoe'), ('Beautiful and the Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('Baby', 'common.topic.notable_types', 'Musical Album'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'), ('PYD', 'common.topic.notable_types', 'Composition'), ('Ashlee Simpson', 'people.person.profession', 'Singer'), ('Pray', 'music.album.artist', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'), ('Trey Songz', 'music.artist.genre', 'Contemporary R&B'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Geri Halliwell', 'people.person.profession', 'Model'), ('iJustine', 'people.person.gender', 'Female'), ('Nelly Furtado', 'people.person.gender', 'Female'), ('Trey Songz', 'people.person.nationality', 'United States of America'), ('m.0ng_vkd', 'film.personal_film_appearance.film', 'A Michael Bublé Christmas'), (\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Ludacris', 'music.composer.compositions', 'Baby'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'), ('Beyoncé Knowles', 'people.person.profession', 'Record producer'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'), ('Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'), ('CL', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Madonna', 'people.person.profession', 'Record producer'), ('m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'), ('1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'), ('PowerHitz', 'broadcast.content.artist', 'M.I.A.'), ('As Long as You Love Me (acoustic version)', 'music.recording.song', 'As Long as You Love Me'), ('Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'), ('Blu Cantrell', 'people.person.gender', 'Female'), ('Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'), ('Rob Thomas', 'people.person.gender', 'Male'), ('Singer', 'people.profession.specializations', 'Piano Singer'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'), ('NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'), ('m.0hvlt03', 'film.film_film_distributor_relationship.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'), ('Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_srv2b'), ('Terence Dudley', 'people.person.profession', 'Musician'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0101fszs', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Official website'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jenna Andrews'), ('FLOW 103', 'broadcast.content.artist', 'Cherish'), ('Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'), ('Next to You', 'music.recording.song', 'Next to You'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'), ('Ray J', 'people.person.nationality', 'United States of America'), ('Usher', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Madonna', 'influence.influence_node.influenced', 'Whitney Houston'), ('m.0w3gbtv', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Montell Jordan', 'music.artist.genre', 'Hip hop music'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Fabolous', 'broadcast.artist.content', 'PowerHitz'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jay-Z', 'common.topic.notable_types', 'Musical Artist'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Max Martin', 'freebase.valuenotation.has_value', 'Parents'), ('Record producer', 'common.topic.webpage', 'm.09ygb05'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'), ('m.0gbm3fj', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Iggy Azalea', 'music.artist.genre', 'Synthpop'), ('Tricky Stewart', 'common.topic.notable_types', 'Record Producer'), ('As Long As You Love Me (Ferry Corsten club dub)', 'common.topic.notable_types', 'Musical Recording'), ('#thatPOWER', 'music.album.album_content_type', 'Studio album'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Katy Perry', 'music.artist.genre', 'Electronic dance music'), ('Kid Cudi', 'people.person.profession', 'Record producer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'), ('m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Jaden Smith', 'people.person.profession', 'Dancer'), ('m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'), ('Keyshia Cole', 'people.person.profession', 'Record producer'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98y7'), ('Person', 'type.type.properties', 'Spouse (or domestic partner)'), ('Fall Out Boy', 'music.artist.origin', 'Chicago'), ('Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'), ('Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'), ('FLOW 103', 'broadcast.content.artist', '50 Cent'), ('Jordin Sparks', 'music.artist.genre', 'Dance-pop'), ('L.A. Reid', 'music.producer.releases_produced', 'My World'), ('L.A. Reid', 'people.person.gender', 'Male'), ('Jessie J', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'), ('1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('London', 'location.location.containedby', 'Ontario'), ('Justin Bieber: Never Say Never', 'film.film.film_set_decoration_by', 'Lia Roldan'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chris Brown', 'music.composer.compositions', 'Next to You'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'), ('Children', 'type.property.schema', 'Person'), ('Change Me', 'music.album.releases', 'Change Me'), ('RedOne', 'music.artist.label', 'Island Records'), ('School Gyrls', 'film.film.starring', 'm.0jztshx'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), ('m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Teen Choice Award for Choice Twitter Personality', 'award.award_category.winners', 'm.0wjhc6c'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'), ('m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Will Smith'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Lupe Fiasco'), ('Hikaru Utada', 'music.artist.label', 'Island Records'), ('Dr. Dre', 'people.person.profession', 'Record producer'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'music.composer.compositions', 'Change Me'), ('Right Here', 'common.topic.notable_types', 'Composition'), ('Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Height'), ('#Thatpower', 'music.recording.artist', 'Will i Am'), ('Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'), ('m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'), ('Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('m.0v90skf', 'award.award_honor.award', 'Billboard Music Award for Top Male Artist'), ('Ludacris', 'people.person.profession', 'Actor'), ('Heartbreaker', 'music.album.genre', 'Pop music'), ('Cameo appearance', 'tv.special_tv_performance_type.episode_performances', 'm.0v1lwt2'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teen idol', 'common.topic.webpage', 'm.0b47zvy'), ('1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'), ('Model', 'base.lightweight.profession.similar_professions', 'Actor'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'), ('Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'One Chance'), ('Never Let You Go', 'common.topic.notable_types', 'Composition'), ('Live My Life', 'common.topic.article', 'm.0j4453y'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'), ('Roller Coaster', 'award.award_nominated_work.award_nominations', 'm.0_x4zg3'), ('Chris Brown', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'), ('Lupe Fiasco', 'music.artist.genre', 'Hip hop music'), ('Teen pop', 'common.topic.article', 'm.02ny8z'), ('PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'), ('Iggy Azalea', 'people.person.gender', 'Female'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Adrienne Bailon', 'people.person.profession', 'Dancer'), ('Hip hop music', 'broadcast.genre.content', '181-beat'), ('m.0sgk_cw', 'award.award_honor.award', \"Kids' Choice Award for Favorite Song\"), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'), ('Iggy Azalea', 'music.artist.genre', 'Electronic dance music'), ('MTV Video Music Brazil Award for Best International Artist', 'award.award_category.winners', 'm.0yrhhqv'), ('Mariah Carey', 'music.artist.label', 'Island Records'), ('Music', 'common.topic.subject_of', 'POPPMusic.net'), ('Camagüey', 'common.topic.notable_types', 'City/Town/Village'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Favorite Girl', 'music.album.artist', 'Justin Bieber'), ('m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'), ('Britney Spears', 'people.person.profession', 'Singer'), ('Die in Your Arms', 'music.recording.song', 'Die in Your Arms'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'), ('Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Rock music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chef Tone', 'music.artist.genre', 'Hip hop music'), ('Rudolph Isley', 'people.person.gender', 'Male'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Barry Weiss'), ('Beauty and a Beat (Bisbetic Instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0z1scxk'), ('Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'), ('Will Smith', 'people.person.profession', 'Actor'), ('Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Fabian', 'people.person.gender', 'Male'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'), ('Somebody to Love (remix)', 'music.album.primary_release', 'Somebody to Love (remix)'), ('HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'), ('Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Height'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'), ('Spouse', 'type.property.expected_type', 'Person'), ('m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'), ('Baby', 'music.recording.artist', 'Ludacris'), ('Rudolph Valentino', 'people.person.nationality', 'United States of America'), ('Hit-Boy', 'music.artist.genre', 'Hip hop music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPower', 'music.composition.recordings', '#Thatpower'), (\"Justin Bieber's Believe\", 'base.schemastaging.context_name.pronunciation', 'm.011h9_22'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'), ('Miley Cyrus', 'people.person.profession', 'Musician'), ('Justin Timberlake', 'people.person.gender', 'Male'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('m.0vp8rhw', 'music.recording_contribution.album', 'Beauty and a Beat (Remixes)'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'), ('Katy Perry: Part of Me', 'common.topic.notable_types', 'Award-Winning Work'), ('m.0jsmvv5', 'film.film_regional_release_date.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'), ('My Worlds: The Collection', 'music.album.release_type', 'Album'), ('All Around The World (featuring Ludacris)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'), ('1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'), ('m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'), ('Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'), ('School Boy Records', 'music.record_label.artist', 'Madison Beer'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'), ('Musical Album', 'freebase.type_hints.included_types', 'Topic'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien dubstep mix)'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Fergie', 'people.person.profession', 'Actor'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Stuart Ford', 'people.person.profession', 'Film Producer'), ('Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'), ('Zac Efron', 'people.person.gender', 'Male'), ('P!nk', 'music.artist.genre', 'Rock music'), ('R. Kelly', 'people.person.profession', 'Film Producer'), ('Gender', 'type.property.schema', 'Person'), ('Adam Messinger', 'music.artist.genre', 'Rhythm and blues'), ('Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'), ('Right Here', 'common.topic.notable_for', 'g.12h31mb_7'), ('JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Jessie J', 'influence.influence_node.influenced', 'Yves Bole'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Slick Rick'), ('Amerie', 'music.artist.genre', 'Rock music'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0pbzq13', 'film.performance.special_performance_type', 'Cameo appearance'), ('Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Height', 'type.property.unit', 'Meter'), ('Iggy Azalea', 'people.person.profession', 'Model'), ('NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'), ('Ray J', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Yves Bole', 'base.svocab.music_artist.genre', 'Pop'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Boyfriend (acoustic version)', 'music.recording.canonical_version', 'Boyfriend'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Believe Tour', 'music.concert_tour.album_or_release_supporting', 'Believe'), ('m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Believe Acoustic', 'music.album.release_type', 'Album'), ('Diplo', 'freebase.valuenotation.has_value', 'Height'), ('Hikaru Utada', 'music.artist.genre', 'Synthpop'), ('Roller Coaster', 'music.composition.composer', 'Julian Swirsky'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.song', 'As Long as You Love Me'), ('Elvis Presley', 'music.artist.genre', 'Pop music'), ('Lady Gaga', 'music.artist.genre', 'Pop music'), ('FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'), ('Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'), ('Usher', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Kelis', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0gbm3b7', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Twista', 'broadcast.artist.content', '.977 The Hits Channel'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'), ('All That Matters', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Nasri', 'music.artist.genre', 'Reggae'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'), ('m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'), ('Bad 25', 'film.film.genre', 'Documentary film'), ('Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'), ('Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me'), ('Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Geri Halliwell', 'people.person.profession', 'Musician'), ('Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'), ('Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Coldplay', 'music.artist.genre', 'Rock music'), ('Kevin Risto', 'people.person.profession', 'Record producer'), ('Whitney Houston', 'people.person.profession', 'Model'), ('Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'), ('Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'), ('181-beat', 'broadcast.content.artist', 'Cassie Ventura'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Disney Parks Christmas Day Parade', 'common.topic.notable_types', 'Award-Winning Work'), ('Ray J', 'people.person.profession', 'Artist'), ('Avril Lavigne', 'people.person.profession', 'Singer-songwriter'), ('American Music Award for Favorite Pop/Rock Male Artist', 'award.award_category.winners', 'm.0ndc0sf'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Music', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Christina Aguilera', 'music.artist.genre', 'Electronic music'), ('PowerHitz', 'broadcast.content.artist', 'Outkast'), ('U Smile', 'music.music_video.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.genre', 'Rock music'), ('Sean Kingston', 'music.artist.genre', 'Hip hop music'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Haley James Scott', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('Kylie Minogue', 'music.artist.genre', 'Rock music'), ('Chris Jasper', 'people.person.nationality', 'United States of America'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My Worlds: The Collection', 'music.album.album_content_type', 'Compilation album'), ('Lolly', 'music.album.releases', 'Lolly'), ('Toby Gad', 'common.topic.notable_types', 'Record Producer'), ('That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'), ('1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'), ('m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'), ('1Club.FM: Power', 'broadcast.content.artist', 'DMX'), ('Ja Rule', 'people.person.profession', 'Singer'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'), ('Die in Your Arms', 'award.award_nominated_work.award_nominations', 'm.0z85qxq'), ('Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'), ('m.012nv5gz', 'people.place_lived.location', 'Camagüey'), ('Kuk Harrell', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft5f'), ('Somebody to Love (J Stax remix)', 'music.recording.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Allison Kaye Scarinzi'), ('Adam Messinger', 'people.person.nationality', 'Canada'), ('Nasri', 'music.artist.genre', 'Pop music'), ('#thatPower', 'music.recording.featured_artists', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'), ('C1', 'common.topic.notable_types', 'Musical Artist'), ('.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'), ('School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'), ('Country', 'freebase.type_profile.strict_included_types', 'Topic'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.05sp405', 'organization.organization_relationship.child', 'Island Records'), ('Savan Kotecha', 'people.person.profession', 'Record producer'), ('Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'), ('m.0b47zvy', 'common.webpage.topic', 'Teen idol'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Teen Choice Award for Choice Summer Music Star: Male', 'award.award_category.winners', 'm.0yrjynf'), ('Juicy J', 'people.person.profession', 'Actor'), ('m.0yqflrk', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('1Club.FM: Power', 'broadcast.content.artist', 'Eminem'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('m.04q65lb', 'organization.organization_relationship.child', 'The Island Def Jam Music Group'), ('Big Sean', 'people.person.nationality', 'United States of America'), ('Beyoncé Knowles', 'people.person.profession', 'Film Producer'), ('R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'), ('1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'), ('Geri Halliwell', 'people.person.profession', 'Actor'), ('Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('My World', 'music.album.artist', 'Justin Bieber'), ('Don Henley', 'people.person.gender', 'Male'), ('HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'), ('Musician', 'people.profession.specializations', 'Singer'), ('Die in Your Arms', 'music.recording.canonical_version', 'Die in Your Arms'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0njvs9s', 'award.award_honor.award', 'CMT Music Award: Collaborative Video of the Year'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber', 'music.artist.album', 'Turn to You (Mother’s Day Dedication)'), ('Ludacris', 'music.artist.contribution', 'm.0vmyv4w'), ('Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'), ('City/Town/Village', 'freebase.type_profile.strict_included_types', 'Topic'), ('Recovery', 'common.topic.notable_types', 'Musical Recording'), ('Dancer', 'common.topic.notable_types', 'Profession'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('Terence Dudley', 'people.person.gender', 'Male'), ('Baby', 'music.composition.recordings', 'Polka Face'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mr. Sosa & The Yayo'), ('Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Rihanna', 'music.artist.genre', 'Dance music'), ('justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'), ('SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Gender', 'type.property.expected_type', 'Gender'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101fvbf', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0yrhrwc', 'award.award_honor.ceremony', '2011 MTV Video Music Aid Japan'), ('MTV Europe Music Award for Best North American Act', 'award.award_category.winners', 'm.0yrhmll'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'), ('m.0101ft1d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'), ('Ciara', 'people.person.gender', 'Female'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'), ('Britney Spears', 'music.artist.genre', 'Synthpop'), ('Thought of You', 'music.recording.artist', 'Justin Bieber'), ('m.0jzrrqs', 'location.mailing_address.country', 'United States of America'), ('Justin Bieber', 'internet.blogger.blog', 'justinbieber'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Toby Gad', 'people.person.nationality', 'United States of America'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Hot Wired Radio', 'broadcast.content.genre', 'Rock music'), ('m.0z8qqh5', 'award.award_nomination.ceremony', '2010 Teen Choice Awards'), ('50 Cent', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Die in Your Arms', 'common.topic.notable_types', 'Composition'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Height'), ('Change Me', 'award.award_nominated_work.award_nominations', 'm.0_w1gn3'), ('Justin Timberlake', 'broadcast.artist.content', 'Hot Wired Radio'), ('All Around The World', 'common.topic.article', 'm.09c4dl'), ('Europop', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('Country', 'freebase.type_profile.strict_included_types', 'Location'), ('Jason Mraz', 'people.person.profession', 'Singer-songwriter'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rune Reilly Kølsch'), ('Turn to You (Mother’s Day Dedication)', 'common.topic.notable_types', 'Musical Album'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rudolph Isley', 'music.artist.genre', 'Rock music'), ('Singer', 'people.profession.specializations', 'Chansonnier'), ('m.0101ftmm', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Kelis', 'music.artist.genre', 'Electronic dance music'), ('Somebody to Love', 'common.topic.article', 'm.0bmc2qq'), ('Bigger', 'music.composition.lyricist', 'Justin Bieber'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Boyfriend', 'music.composition.composer', 'Matthew Musto'), ('m.0gbm3cx', 'film.personal_film_appearance.person', 'Usher'), ('Roller Coaster', 'common.topic.notable_types', 'Composition'), ('Avril Lavigne', 'music.artist.genre', 'Rock music'), ('The Pussycat Dolls', 'music.artist.genre', 'Rhythm and blues'), ('Right Here', 'common.topic.notable_types', 'Musical Album'), ('Johntá Austin', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'influence.influence_node.influenced', 'Yves Bole'), ('Britney Spears', 'broadcast.artist.content', '1Club.FM: Power'), ('Kelly Clarkson', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('#thatPOWER', 'music.recording.tracks', '#thatpower'), ('Yves Bole', 'base.musicpf.artist.genre', 'Pop music'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('PowerHitz', 'broadcast.content.artist', 'Trick Daddy'), ('Haley James Scott', 'people.person.gender', 'Female'), ('Justin Bieber', 'music.artist.album', 'Believe Acoustic'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'), ('Deborah Lurie', 'people.person.gender', 'Female'), ('Jordan Francis', 'people.person.gender', 'Male'), ('Big R Radio - The Hawk', 'common.topic.article', 'm.03hs0cl'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0wjgrzc'), ('Children', 'rdf-schema#domain', 'Person'), ('Ciara', 'people.person.profession', 'Record producer'), ('Linkin Park', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('All That Matters', 'award.award_nominated_work.award_nominations', 'm.0_vw6nn'), ('First Dance', 'music.composition.composer', 'Alexander Parhm, Jr.'), ('Blackstreet', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Leif Garrett', 'music.artist.genre', 'Pop music'), ('Duffy', 'people.person.gender', 'Female'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Official website'), ('The Island Def Jam Music Group', 'freebase.valuenotation.has_value', 'Founders'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0f0dwc4'), ('HitzRadio.com', 'broadcast.content.artist', 'AnnaGrace'), ('m.0sxhgqj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Live My Life', 'music.composition.recordings', 'Live My Life (Jaywalker remix)'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('Christina Milian', 'people.person.nationality', 'United States of America'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Eminem', 'broadcast.artist.content', '.977 The Hits Channel'), ('Ciara', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Dapo Torimiro', 'people.person.profession', 'Musician'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Gwen Stefani', 'music.artist.genre', 'Rhythm and blues'), ('Love Never Felt So Good', 'music.album.release_type', 'Single'), ('Never Say Never', 'music.recording.canonical_version', 'Never Say Never'), ('CL', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sean Combs', 'people.person.profession', 'Musician'), ('Justin Bieber', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Green Day', 'common.topic.notable_types', 'Musical Artist'), ('Anastacia', 'people.person.profession', 'Actor'), ('As Long as You Love Me', 'music.album.releases', 'As Long as You Love Me'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.episode', 'Tina Fey / Justin Bieber'), ('Roller Coaster', 'common.topic.notable_types', 'Musical Album'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Remix)'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.012m1vz9', 'music.group_membership.role', 'Record producer'), ('Lil Jon', 'common.topic.notable_types', 'Musical Artist'), ('Nelly Furtado', 'music.artist.genre', 'Dance music'), ('FLOW 103', 'broadcast.content.artist', 'Justin Timberlake'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Nikki Williams'), ('Selena Gomez', 'music.artist.genre', 'Dance-pop'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('m.0gbcs16', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0y5th3r', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Live My Life', 'music.album.featured_artists', 'Justin Bieber'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('PowerHitz', 'broadcast.content.artist', 'Chingy'), ('Jessie J', 'people.person.profession', 'Musician'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Hip hop music'), ('Lady Gaga', 'broadcast.artist.content', 'HitzRadio.com'), (\"Destiny's Child\", 'music.artist.genre', 'Rhythm and blues'), ('Elvis Presley', 'people.person.languages', 'English Language'), ('Beautiful', 'music.composition.lyricist', 'Alex Lambert'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Redfoo', 'music.artist.genre', 'Pop music'), ('Beyoncé Knowles', 'broadcast.artist.content', '181-beat'), ('David Cassidy', 'people.person.languages', 'English Language'), ('Jason Mraz', 'broadcast.artist.content', 'Sunshine Radio'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'people.person.gender', 'Female'), ('m.012bm2v1', 'celebrities.friendship.friend', 'iJustine'), ('The Island Def Jam Music Group', 'organization.organization.leadership', 'm.0ncpx3v'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftk6'), ('Never Say Never', 'music.album.contributor', 'm.0vp7qr5'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Justin Bieber: Never Say Never', 'film.film.release_date_s', 'm.0jsmvv5'), ('One Time', 'common.topic.article', 'm.0k20sjb'), ('My World', 'music.album.releases', 'My World'), ('Little Bird', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Aaliyah', 'music.artist.genre', 'Rock music'), ('Benny Blanco', 'music.artist.genre', 'Hip hop music'), ('Madonna', 'music.artist.genre', 'Rock music'), ('Bad Day', 'music.composition.recordings', 'Bad Day'), ('Demi Lovato', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('My Worlds Acoustic', 'music.album.releases', 'My Worlds Acoustic'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Leona Lewis'), ('Pearl Jam', 'influence.influence_node.influenced', 'Kings of Leon'), ('Britney Spears', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('My Worlds', 'music.album.album_content_type', 'Compilation album'), ('First Dance', 'music.composition.recordings', 'First Dance'), ('#thatPOWER', 'music.album.primary_release', '#thatPOWER'), ('Caitlin Beadles', 'base.popstra.celebrity.dated', 'm.0gxnp0c'), ('Beauty and a Beat (Wideboys Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Janet Jackson', 'people.person.profession', 'Dancer'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO RNB Mix'), ('HitzRadio.com', 'broadcast.content.genre', 'Rap music'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Rap music'), ('m.0yrk18w', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Next to You', 'common.topic.notable_for', 'g.126smxx6r'), ('Contemporary R&B', 'music.genre.recordings', 'Daforce Is Wit Me'), ('m.0101fvj9', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('m.0j_tkcq', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Jordin Sparks', 'people.person.profession', 'Singer'), ('Frank Ocean', 'music.composer.compositions', 'Bigger'), ('2013 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('Madonna', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('m.0gfpbq5', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('Jordan Francis', 'people.person.profession', 'Singer-songwriter'), ('Green Day', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Timbaland', 'music.artist.genre', 'Rap music'), ('2013 MTV Europe Music Awards', 'time.event.people_involved', 'Redfoo'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Dance music'), ('Jane Lipsitz', 'film.producer.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Garrett Grant'), ('Disney Parks Christmas Day Parade', 'film.film.personal_appearances', 'm.0v_714c'), ('181-beat', 'broadcast.content.artist', 'Akon'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sonique'), ('Hip hop music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('4th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0y_g556'), ('Whitney Houston', 'music.artist.genre', 'Pop music'), ('Janet Jackson', 'music.artist.genre', 'Contemporary R&B'), ('Alanis Morissette', 'music.artist.genre', 'Rock music'), ('Kylie Minogue', 'music.artist.genre', 'Dance music'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Radio Mix)'), ('Haley James Scott', 'fictional_universe.fictional_character.gender', 'Female'), ('Mariah Carey', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0njw444', 'award.award_honor.ceremony', '2011 MTV Europe Music Awards'), ('Teyana', 'music.artist.genre', 'Contemporary R&B'), ('Kylie Minogue', 'people.person.languages', 'English Language'), ('Will i Am', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft1r'), ('HitzRadio.com', 'broadcast.content.artist', 'Lil Jon'), ('m.0njhyh_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Christina Aguilera', 'people.person.profession', 'Dancer'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('One Time', 'music.album.release_type', 'Single'), ('Taylor Swift', 'people.person.profession', 'Singer-songwriter'), ('Iggy Azalea', 'people.person.profession', 'Singer'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock Remix)'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njdns_'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Top 40'), ('Chicago', 'base.biblioness.bibs_topic.is_really', 'Chicago'), ('FLOW 103', 'broadcast.content.genre', 'Pop music'), ('Lolly', 'music.single.versions', 'Lolly'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Actor'), ('Justin Bieber', 'people.person.parents', 'Pattie Mallette'), ('Soulja Boy', 'people.person.nationality', 'United States of America'), ('Next to You', 'music.recording.artist', 'Justin Bieber'), ('Journals', 'common.topic.notable_for', 'g.1ypm_b95r'), ('Alicia Keys', 'music.artist.genre', 'Electronic music'), ('All Bad', 'music.composition.composer', 'Ryan Toby'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ashlee Simpson', 'people.person.gender', 'Female'), ('The Island Def Jam Music Group', 'common.topic.notable_types', 'Record label'), ('Snoop Dogg', 'music.artist.genre', 'Contemporary R&B'), ('Katy Perry', 'film.producer.film', 'Katy Perry: Part of Me'), ('Rob Thomas', 'people.person.profession', 'Singer-songwriter'), ('L.A. Reid', 'film.producer.film', 'School Gyrls'), ('181-beat', 'broadcast.content.artist', 'Mario Winans'), ('Shaffer Smith', 'broadcast.artist.content', 'PowerHitz'), ('Juvenile', 'music.artist.genre', 'Hip hop music'), ('m.0v1lwt2', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Eminem', 'people.person.profession', 'Film Producer'), ('RedOne', 'people.person.gender', 'Male'), ('50 Cent', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98y7'), ('Victoria Justice', 'people.person.nationality', 'United States of America'), ('Kanye West', 'people.person.profession', 'Record producer'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0yr9tjb'), ('A Star Was Born: Justin Bieber', 'film.film.personal_appearances', 'm.0h4yhbb'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y803nt'), ('Justin Bieber', 'book.author.works_written', 'Justin Bieber: Just Getting Started'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Cascada'), ('.977 The Hits Channel', 'broadcast.content.artist', 'P!nk'), ('Yves Bole', 'people.person.parents', 'Maribel Alonso Diaz'), ('Ciara', 'broadcast.artist.content', 'Hot 108 Jamz'), ('My World 2.0', 'music.album.genre', 'Pop music'), ('Never Say Never: The Remixes', 'music.album.genre', 'Pop music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Antebellum'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Height'), ('Yves Bole', 'common.topic.notable_for', 'g.11b75sb023'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y7y53r'), ('m.0y_g42w', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.form', 'Song'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Switchfoot'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvjw'), ('Big R Radio - The Hawk', 'broadcast.content.broadcast', 'Big R Radio - The Hawk - 128kbps Stream'), ('m.0pcr28j', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Marvin Isley', 'people.person.profession', 'Singer'), ('DMX', 'people.person.profession', 'Actor'), ('A Michael Bublé Christmas', 'film.film.language', 'English Language'), ('Musical Artist', 'freebase.type_hints.included_types', 'Topic'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'OneRepublic'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life (Jaywalker remix)'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Timbaland', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.artist.album', 'As Long as You Love Me'), ('First Dance', 'common.topic.image', 'Justin Bieber in concert crop'), ('Rodney Jerkins', 'music.artist.genre', 'Dance music'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Height'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Shaffer Smith'), ('Justin Bieber', 'music.composer.compositions', 'Confident'), ('Nasri', 'music.artist.genre', 'Contemporary R&B'), ('All Around The World (featuring Ludacris)', 'music.recording.canonical_version', 'All Around the World'), ('Eminem', 'people.person.nationality', 'United States of America'), ('m.0sxhq3d', 'award.award_nomination.ceremony', 'Juno Awards of 2013'), ('m.0t_l7mp', 'award.award_nomination.nominated_for', 'Believe'), ('Beyoncé Knowles', 'music.artist.genre', 'Hip hop music'), ('Die in Your Arms', 'common.topic.article', 'm.0jwx2__'), ('Johntá Austin', 'music.composer.compositions', 'Never Let You Go'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfmm45'), ('Height', 'type.property.schema', 'Person'), ('m.0ng_k21', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0ndc0sf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'music.artist.album', 'Right Here'), ('Record producer', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('Fergie', 'people.person.nationality', 'United States of America'), ('Beautiful', 'music.recording.song', 'Beautiful'), ('Jason Mraz', 'broadcast.artist.content', 'radioIO Todays POP'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Wayne Wonder'), ('Under the Mistletoe', 'music.album.genre', 'Reggae'), ('Boyfriend (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Christina Milian', 'music.artist.genre', 'Pop music'), ('m.0jztshx', 'film.performance.film', 'School Gyrls'), ('#Thatpower', 'common.topic.notable_types', 'Musical Recording'), ('Dr. Dre', 'people.person.profession', 'Film Producer'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0z8qx3w'), ('Johntá Austin', 'common.topic.notable_types', 'Musical Artist'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Amerie'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjdc'), ('Sia Furler', 'music.artist.genre', 'Dance-pop'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Fabolous'), ('181-beat', 'broadcast.content.artist', 'Shaffer Smith'), ('Beauty And A Beat', 'music.composition.composer', 'Max Martin'), ('Selena Gomez', 'music.artist.genre', 'Rhythm and blues'), ('Beauty and a Beat', 'music.single.versions', 'Beautiful and the Beat'), ('School Gyrls', 'common.topic.notable_types', 'Film'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftm6'), (\"Justin Bieber's Believe\", 'film.film.edited_by', 'Avi Youabian'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Parents'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Profession'), ('KooL CrAzE', 'common.topic.notable_types', 'Musical Artist'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sir Nolan', 'people.person.profession', 'Musician'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.song', 'As Long as You Love Me'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Instrumental)'), ('Iggy Azalea', 'music.artist.genre', 'Rhythm and blues'), ('Nicki Minaj', 'music.featured_artist.albums', 'Beauty and a Beat'), ('m.04kjw8n', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('m.0njdns_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Ja Rule', 'people.person.nationality', 'United States of America'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ice Cube'), ('Teen idol', 'base.icons.icon_genre.icons', 'Paul Anka'), ('U Smile', 'award.award_winning_work.awards_won', 'm.0n4rmg7'), ('Hot Wired Radio', 'broadcast.content.artist', 'Ciara'), ('Roller Coaster', 'music.composition.recordings', 'Roller Coaster'), ('Barry Weiss', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Fray'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('m.0y803nt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Shaggy', 'people.person.gender', 'Male'), ('Kelis', 'broadcast.artist.content', 'radioIO Todays POP'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Sia Furler', 'music.artist.genre', 'Electronic dance music'), ('Juvenile', 'broadcast.artist.content', '.977 The Hits Channel'), ('Rudolph Isley', 'music.artist.genre', 'Rhythm and blues'), ('Ja Rule', 'music.artist.genre', 'Contemporary R&B'), ('m.0z1jn32', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Santana', 'broadcast.artist.content', 'Sunshine Radio'), ('Billboard Music Milestone Award', 'award.award_category.winners', 'm.0vb6hhj'), ('Selena Gomez', 'people.person.profession', 'Singer'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My World', 'music.album.genre', 'Teen pop'), ('Yves Bole', 'people.person.languages', 'Italian Language'), ('PowerHitz', 'broadcast.content.artist', 'Lady Gaga'), ('m.0gxnp02', 'base.popstra.dated.participant', 'Selena Gomez'), ('Contemporary R&B', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('Boyfriend', 'music.recording.canonical_version', 'Boyfriend'), ('Rudolph Valentino', 'base.icons.icon.icon_genre', 'Teen idol'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0ywsnc9'), ('m.0z1scxk', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Sash!'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Leona Lewis'), ('m.0101ft3d', 'film.personal_film_appearance.person', 'Ellen DeGeneres'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Lifehouse'), ('Willa Ford', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Elvis Presley'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mannie Fresh'), ('1Club.FM: Power', 'broadcast.content.artist', 'Will Smith'), ('Dapo Torimiro', 'music.artist.genre', 'Contemporary R&B'), ('Ja Rule', 'music.artist.genre', 'Pop music'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien Luvstep mix)'), ('Taylor Swift', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.release_date_s', 'm.0j593k1'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('All Around the World (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Alicia Keys', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnt3'), ('m.0_svs8m', 'award.award_nomination.nominated_for', 'justinbieber'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0ndc0sf', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Male Artist'), ('Justin Bieber: First Step 2 Forever', 'common.topic.notable_types', 'Written Work'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Ludacris', 'broadcast.artist.content', '.977 The Hits Channel'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('m.0zbg2kw', 'award.award_nomination.nominated_for', 'My World 2.0'), ('PYD', 'music.composition.composer', 'R. Kelly'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Gender'), ('HitzRadio.com', 'broadcast.content.artist', 'Mims'), ('Wait For a Minute', 'music.recording.artist', 'Justin Bieber'), ('Place of birth', 'type.property.schema', 'Person'), ('Beyoncé Knowles', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: 80s (Pop)', 'broadcast.content.producer', '1Club.FM'), ('m.0gctytd', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Akon', 'people.person.gender', 'Male'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Katy Perry', 'people.person.profession', 'Musician'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('U Smile', 'common.topic.notable_types', 'Award-Winning Work'), ('Mannie Fresh', 'people.person.gender', 'Male'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('Never Say Never: The Remixes', 'common.topic.notable_types', 'Musical Album'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rodney Jerkins', 'freebase.valuenotation.has_value', 'Parents'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg8bc1'), ('Mary J. Blige', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'World music'), ('Donna Summer', 'people.person.nationality', 'United States of America'), ('Pop music', 'music.genre.parent_genre', 'Rock music'), ('Johntá Austin', 'music.artist.genre', 'Pop music'), ('Leonardo DiCaprio', 'people.person.gender', 'Male'), ('m.0j2gfc1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Hot Wired Radio', 'broadcast.content.artist', 'Will i Am'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Journals', 'music.album.releases', 'Journals'), ('Rihanna', 'broadcast.artist.content', '.977 The Hits Channel'), ('Terence Dudley', 'music.artist.genre', 'Hip hop music'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Hip hop music', 'common.topic.subjects', 'Brian Keith Kennedy'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'First Dance'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Island Records', 'music.record_label.releases', 'Believe'), ('m.0z8qqh5', 'award.award_nomination.nominated_for', 'My World 2.0'), ('David Nicksay', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'music.artist.album', 'Wait For a Minute'), ('As Long As You Love Me (Audiobot edit)', 'common.topic.notable_types', 'Musical Recording'), ('Will i Am', 'people.person.profession', 'Film Producer'), ('Nyíregyháza', 'common.topic.notable_types', 'City/Town/Village'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv62'), ('HitzRadio.com', 'broadcast.content.artist', 'Jason Mraz'), ('The Notorious B.I.G.', 'people.person.profession', 'Artist'), ('Will i Am', 'people.person.profession', 'Musician'), ('Mariah Carey', 'music.artist.genre', 'Dance music'), ('Confident', 'music.composition.composer', 'Chance the Rapper'), ('m.0tkc3tj', 'award.award_nomination.nominated_for', 'My World'), ('Selena Gomez', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.composition.composer', 'Bilal Hajji'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All That Matters', 'music.composition.composer', 'Donovan Knight'), ('m.09ygb05', 'common.webpage.topic', 'Record producer'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Stratford', 'common.topic.notable_types', 'City/Town/Village'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.composer', 'Adam Messinger'), ('Ray J', 'music.artist.genre', 'Pop music'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Nasri', 'music.composer.compositions', 'Next to You'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6_81'), ('Die in Your Arms', 'music.album.releases', 'Die in Your Arms'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Panic! at the Disco'), ('#thatPower', 'music.composition.recordings', '#Thatpower'), ('m.0101fvc3', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Green Day'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Jayceon Terrell Taylor', 'music.artist.genre', 'Hip hop music'), ('m.0ndc3_1', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0tjyljn', 'award.award_nomination.ceremony', 'Juno Awards of 2011'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Weezer'), ('Jordan Pruitt', 'people.person.gender', 'Female'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot edit)'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'TLC'), ('Michael Jackson', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0z1scxk'), ('Mariah Carey', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Timberlake', 'broadcast.artist.content', '1.FM Top 40'), ('We Are the World 25 for Haiti', 'music.album.release_type', 'Single'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'common.topic.notable_types', 'Film'), ('Hikaru Utada', 'people.person.gender', 'Female'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)'), ('Justin Bieber', 'music.composer.compositions', 'Boyfriend'), ('Big R Radio - Top 40 Hits', 'common.topic.image', 'Big R radio'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juelz Santana', 'people.person.profession', 'Record producer'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Pop music', 'common.topic.subjects', 'Bleona'), ('Electronic dance music', 'music.genre.subgenre', 'Eurodance'), ('Beyoncé Knowles', 'broadcast.artist.content', '1.FM Top 40'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Sean Paul'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h5m2b'), ('Shaggy', 'people.person.nationality', 'United States of America'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0h4yhbb', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0sgkrj4', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jay Cassidy', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.song', 'As Long as You Love Me'), ('Tupac Shakur', 'music.artist.genre', 'Hip hop music'), ('Ray J', 'people.person.gender', 'Male'), ('Justin Bieber', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Tricky Stewart', 'people.person.gender', 'Male'), ('m.0njvvj_', 'award.award_honor.ceremony', 'Juno Awards of 2012'), ('Eminem', 'broadcast.artist.content', 'Sunshine Radio'), ('Live My Life (Party Rock remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('Zendaya: Behind the Scenes', 'film.film.genre', 'Documentary film'), ('Hikaru Utada', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0vb6hhj'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvph'), ('Vanessa Hudgens', 'music.artist.genre', 'Dance music'), ('m.0j2gfc1', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Geri Halliwell', 'broadcast.artist.content', '1.FM Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074l61'), ('Christina Aguilera', 'broadcast.artist.content', 'radioIO Todays POP'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Boys Like Girls'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Never Let You Go', 'music.composition.composer', 'Johntá Austin'), ('Johntá Austin', 'people.person.profession', 'Singer'), ('m.0yrk0mt', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('Live My Life', 'music.composition.composer', 'Martin Kierszenbaum'), ('Live My Life', 'music.composition.composer', 'RedOne'), ('Synthpop', 'music.genre.parent_genre', 'Pop music'), ('Janet Jackson', 'people.person.profession', 'Film Producer'), ('1.FM Top 40', 'broadcast.content.artist', 'Twista'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k3w'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvps'), ('Composition', 'freebase.type_hints.included_types', 'Topic'), ('radioIO Classic RNB', 'common.topic.image', 'RadioIO.png'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me (album version)'), ('Confident', 'common.topic.notable_types', 'Composition'), ('radioIO Todays POP', 'common.topic.article', 'm.03hs0q2'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'All Around The World'), ('1.FM Top 40', 'broadcast.content.genre', 'Classic hits'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('m.0gbm35z', 'film.film_cut.film', 'Justin Bieber: Never Say Never'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('justinbieber', 'internet.blog.language', 'English Language'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Aaliyah'), ('RedOne', 'music.composer.compositions', 'Live My Life'), ('Victoria Justice', 'people.person.profession', 'Musician'), ('Wait For a Minute', 'common.topic.notable_types', 'Musical Recording'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0y_g42w', 'award.award_nomination.ceremony', '4th Annual Shorty Awards'), ('Shorty Award for Music', 'award.award_category.winners', 'm.0z0tmyv'), ('United States of America', 'location.location.time_zones', 'Eastern Time Zone'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Radio Mix)'), ('m.0ywsnc9', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Somebody to Love', 'music.album.release_type', 'Single'), ('Yves Bole', 'influence.influence_node.influenced', 'Lady Gaga'), ('Ciara', 'people.person.profession', 'Dancer'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Official website'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'common.topic.notable_types', 'Musical Recording'), ('HitzRadio.com', 'broadcast.content.artist', 'Beyoncé Knowles'), ('As Long As You Love Me (Audien dubstep edit)', 'common.topic.notable_types', 'Musical Recording'), ('Savan Kotecha', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Journals', 'music.album.artist', 'Justin Bieber'), ('Change Me', 'music.composition.recordings', 'Change Me'), ('Whitney Houston', 'influence.influence_node.influenced', 'Brandy Norwood'), ('Don Henley', 'common.topic.notable_types', 'Musical Artist'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Place of birth'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('P!nk', 'music.artist.genre', 'Dance-pop'), ('m.0101fszb', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Jay-Z', 'music.artist.label', 'The Island Def Jam Music Group'), ('Demi Lovato', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.album', 'Beauty and a Beat'), ('1.FM Top 40', 'broadcast.content.artist', 'Mariah Carey'), ('m.0z85n_1', 'award.award_nomination.nominated_for', 'All Around The World'), ('Where Are Ü Now', 'common.topic.notable_types', 'Musical Recording'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Encore'), ('Jay-Z', 'people.person.profession', 'Record producer'), ('Max Martin', 'people.person.gender', 'Male'), ('Contemporary R&B', 'music.genre.parent_genre', 'Arabic pop music'), ('m.0n58kgb', 'award.award_nomination.award', 'American Music Award for Favorite Pop/Rock Album'), ('Heartbreaker', 'common.topic.notable_types', 'Composition'), ('R. Kelly', 'music.artist.genre', 'Dance-pop'), ('Verse Simmonds', 'freebase.valuenotation.has_value', 'Date of birth'), ('Terius Nash', 'music.artist.genre', 'Synthpop'), ('Estelle', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.012nh7h_', 'theater.theater_role.role', 'Singer'), ('Jaden Smith', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('PowerHitz', 'broadcast.content.genre', 'Urban contemporary'), ('Hot 108 Jamz', 'broadcast.content.artist', 'DMX'), ('Jordan Pruitt', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Dany Brillant', 'common.topic.notable_types', 'Musical Artist'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Fray'), ('Justin Bieber', 'music.composer.compositions', '#thatPower'), ('Michael Jackson', 'people.person.profession', 'Singer-songwriter'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Sir Mix-a-Lot', 'people.person.gender', 'Male'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Alicia Keys', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Channel One', 'broadcast.content.location', 'Chicago'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9j0d'), ('m.046c33q', 'common.webpage.category', 'Topic Webpage'), ('Baby', 'music.composition.recordings', 'Baby'), ('Believe Tour', 'common.topic.notable_types', 'Concert tour'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Singer', 'people.profession.specializations', 'Carnatic Singer'), ('William Orbit', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Baby', 'music.composition.recordings', 'Baby'), ('Hikaru Utada', 'people.person.nationality', 'United States of America'), ('Die in Your Arms', 'common.topic.notable_types', 'Musical Album'), ('Never Say Never', 'music.recording.featured_artists', 'Jaden Smith'), ('Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Record producer', 'common.topic.webpage', 'm.0b497pt'), ('m.012bm3j9', 'celebrities.friendship.friend', 'Justin Bieber'), ('Justin Bieber', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sean Combs'), ('m.0gbmntw', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011c0k9z'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Beyoncé Knowles', 'music.artist.genre', 'Rhythm and blues'), ('Sunshine Radio', 'common.topic.image', 'sunshine_radio.gif'), ('Keyshia Cole', 'music.artist.genre', 'Hip hop music'), ('Scottish Canadian', 'people.ethnicity.included_in_group', 'Canadian'), ('1.FM Top 40', 'common.topic.article', 'm.03hs08z'), ('P!nk', 'people.person.profession', 'Dancer'), ('L.A. Reid', 'music.artist.genre', 'Hip hop music'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_706c'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Flyleaf'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkyfg'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Urban contemporary'), ('Frank Ocean', 'people.person.gender', 'Male'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Three Days Grace'), ('Little Bird', 'common.topic.notable_for', 'g.1yg9546vy'), ('m.0101fvmb', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0nhfd4m', 'award.award_nomination.nominated_for', 'Believe'), ('Max Martin', 'music.artist.genre', 'Rock music'), ('Diplo', 'people.person.nationality', 'United States of America'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Terence Dudley'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Kelly Clarkson', 'broadcast.artist.content', 'Hot Wired Radio'), ('Rodney Jerkins', 'music.artist.genre', 'Contemporary R&B'), ('Runaway Love (remix)', 'common.topic.notable_for', 'g.1259h0ss3'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Change Me', 'common.topic.notable_types', 'Composition'), ('Trick Daddy', 'music.artist.genre', 'Hip hop music'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('m.010htdc2', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot remix)'), ('Lady Gaga', 'music.artist.genre', 'Eurodance'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Height'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Date of birth'), ('Donna Summer', 'people.person.languages', 'English Language'), ('United States of America', 'location.country.languages_spoken', 'Spanish Language'), ('Believe', 'award.award_winning_work.awards_won', 'm.0ndc259'), ('Big R Radio - Top 40 Hits', 'broadcast.content.broadcast', 'Big R Radio - Top 40 Hits - 128kbps Stream'), ('Spouse', 'rdf-schema#range', 'Person'), ('Nelly', 'people.person.nationality', 'United States of America'), ('Billboard Music Award for Top Pop Album', 'award.award_category.winners', 'm.0njhxzc'), ('Tupac Shakur', 'people.person.profession', 'Writer'), ('Billboard Music Award for Top New Artist', 'award.award_category.winners', 'm.0njhtjj'), ('m.0z0tmyv', 'award.award_honor.honored_for', 'justinbieber'), ('m.0wjgqck', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('181-beat', 'broadcast.content.artist', 'Nelly'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Madonna', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('The Isley Brothers', 'music.artist.genre', 'Pop music'), ('m.0gwhmhm', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0zg8bc1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Will Smith', 'broadcast.artist.content', '1.FM Top 40'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.episode', 'Meet The Cast & Results Show: Week 1'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('Diplo', 'music.artist.genre', 'Reggae'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'music.composition.recordings', 'Beautiful'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Nick Jonas', 'people.person.gender', 'Male'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot edit)'), ('m.0101ft3d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Snoop Dogg'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Children'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_sxby0'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('HitzRadio.com', 'common.topic.notable_for', 'g.1254y3j6d'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Timbaland'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gbm3fr', 'film.personal_film_appearance.person', 'Scooter Braun'), ('The Isley Brothers', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Fall Out Boy', 'broadcast.artist.content', 'radioIO Todays POP'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y_g556'), ('iJustine', 'people.person.nationality', 'United States of America'), ('Barry Weiss', 'people.person.profession', 'Music executive'), ('Justin Bieber', 'music.artist.album', 'Baby'), ('HitzRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.episode', 'True Concert'), ('2011 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('As Long as You Love Me (album version)', 'music.recording.artist', 'Justin Bieber'), ('Snoop Dogg', 'music.artist.genre', 'Reggae'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Christina Aguilera'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO Todays RNB'), ('Christina Milian', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0t_l7mp', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Everlast', 'people.person.profession', 'Actor'), ('Anastacia', 'music.artist.genre', 'Rock music'), ('Ginuwine', 'people.person.gender', 'Male'), ('Jason Mraz', 'music.artist.genre', 'Pop music'), ('Baby', 'common.topic.notable_for', 'g.12h2w5zfr'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01068tkg'), ('m.0v1lwt2', 'tv.tv_guest_role.episodes_appeared_in', 'The Fabulous Faker Boy'), ('Juvenile', 'common.topic.notable_types', 'Musical Artist'), ('DMX', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Christina Aguilera', 'broadcast.artist.content', 'Sunshine Radio'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk0mt'), ('R. Kelly', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0njw59_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Children'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vb6hhj', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92x'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.artist', 'Justin Bieber'), ('Yves Bole', 'people.person.places_lived', 'm.012nv5gz'), ('Wait For a Minute', 'music.recording.artist', 'Tyga'), ('Believe', 'music.album.genre', 'Dance-pop'), ('Redfoo', 'people.person.profession', 'Record producer'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0pc67ly', 'award.award_nomination.ceremony', '2011 MTV Movie Awards'), ('Justin Bieber', 'music.composer.compositions', 'Eenie Meenie'), ('Kylie Minogue', 'broadcast.artist.content', 'radioIO Todays POP'), ('Mason Levy', 'freebase.valuenotation.has_value', 'Place of birth'), ('Jordan Francis', 'freebase.valuenotation.has_value', 'Parents'), ('HitzRadio.com', 'broadcast.content.artist', 'NB Ridaz'), ('Justin Bieber', 'music.featured_artist.albums', 'Next to You'), ('m.0njw1tn', 'award.award_honor.award_winner', 'Justin Bieber'), ('My World 2.0', 'common.topic.article', 'm.09v5jd_'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Miley Cyrus'), ('The Island Def Jam Music Group', 'organization.organization.headquarters', 'm.0jzrrqs'), (\"2011 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkyfg'), ('Justin Bieber Pictures', 'common.resource.annotations', 'm.0bvmhvb'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v1d2xz', 'award.award_nomination.nominated_for', 'Boyfriend'), ('Contemporary R&B', 'broadcast.genre.content', '181-rnb'), ('Anastacia', 'people.person.profession', 'Musician'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Mannie Fresh', 'people.person.profession', 'Record producer'), ('m.0h4yhbb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('#thatPower', 'common.topic.notable_for', 'g.11b5m0k2ty'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Avery'), ('Miley Cyrus', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Jordin Sparks', 'people.person.gender', 'Female'), ('Bruce Dickinson', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0ywtfj6'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0yr9tjb', 'award.award_nomination.nominated_for', 'My World'), ('Timbaland', 'music.artist.genre', 'Synthpop'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('m.03hs4ny', 'common.webpage.category', 'Topic Webpage'), ('Soulja Boy', 'people.person.profession', 'Record producer'), ('Snoop Dogg', 'music.artist.genre', 'Rhythm and blues'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Children'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Never Say Never', 'music.album.releases', 'Never Say Never'), ('Ernie Isley', 'music.composer.compositions', 'Bad Day'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0zg8bc1'), ('Nicki Minaj', 'people.person.gender', 'Female'), ('Ginuwine', 'music.artist.genre', 'Dance music'), ('Bad 25', 'film.film.country', 'United States of America'), ('Heartbreaker', 'music.recording.artist', 'Justin Bieber'), ('Christina Aguilera', 'people.person.profession', 'Film Producer'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Musical Artist', 'freebase.type_profile.strict_included_types', 'Topic'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Isley Brothers'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Gender'), ('NME Awards 2012', 'award.award_ceremony.awards_presented', 'm.0z8755b'), ('2014 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.010lkp2z'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Love Never Felt So Good', 'music.composition.recorded_as_album', 'Love Never Felt So Good'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('First Dance', 'music.recording.featured_artists', 'Usher'), ('My World 2.0', 'common.topic.image', 'justin'), ('m.0gbm3f5', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', \"Destiny's Child\"), ('Under the Mistletoe', 'music.album.genre', 'Contemporary R&B'), ('1Club.FM: V101', 'broadcast.content.artist', 'Craig David'), ('Tommy Sands', 'people.person.profession', 'Singer'), ('Start Date', 'type.property.expected_type', 'Date/Time'), ('Jaxon Bieber', 'people.person.gender', 'Male'), ('Date of birth', 'rdf-schema#range', 'Date/Time'), ('Akon', 'broadcast.artist.content', '181-beat'), ('Savan Kotecha', 'freebase.valuenotation.has_value', 'Date of birth'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012bm3j9'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Record producer', 'common.topic.article', 'm.0dz40'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Van Halen', 'broadcast.artist.content', '1.FM Top 40'), ('m.0z2dr9y', 'award.award_nomination.award', 'Shorty Award for Music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Colbie Caillat'), ('Avril Lavigne', 'broadcast.artist.content', 'Big R Radio - The Hawk'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvl8'), ('Christina Aguilera', 'people.person.languages', 'English Language'), ('Profession', 'type.property.schema', 'Person'), ('Linkin Park', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Christina Milian', 'people.person.profession', 'Dancer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Chris Brown'), ('Everlast', 'people.person.nationality', 'United States of America'), ('Pras', 'people.person.profession', 'Singer'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Ashley Tisdale', 'people.person.gender', 'Female'), ('Soulja Boy', 'music.artist.genre', 'Dance-pop'), ('m.0y803nt', 'award.award_honor.award_winner', 'Justin Bieber'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0gj0bj6', 'tv.tv_guest_role.episodes_appeared_in', 'JLS and Justin Bieber @ R1BW'), ('Willa Ford', 'people.person.gender', 'Female'), ('Teen idol', 'common.topic.article', 'm.0mb9c'), ('Runaway Love (remix)', 'music.album.primary_release', 'Runaway Love (remix)'), ('Justin Bieber', 'music.artist.label', 'The Island Def Jam Music Group'), ('Max Martin', 'common.topic.notable_types', 'Record Producer'), ('m.0bldgxn', 'common.resource.annotations', 'm.07lkzw7'), ('Parents', 'rdf-schema#domain', 'Person'), ('Contemporary R&B', 'broadcast.genre.content', 'ProjectVIBE.net'), ('Foreign Remix', 'music.recording.artist', 'Trey Songz'), ('Playback Singer', 'common.topic.notable_types', 'Profession'), ('Justin Timberlake', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Justin Timberlake'), ('Kuk Harrell', 'music.artist.genre', 'Contemporary R&B'), ('Hikaru Utada', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.artist.album', 'Favorite Girl'), ('Baby (acoustic)', 'common.topic.notable_types', 'Musical Recording'), ('Jason Mraz', 'people.person.nationality', 'United States of America'), ('m.0pcr2qn', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0z0tgz6', 'award.award_nomination.award', 'Shorty Award for Music'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot edit)'), ('Ethnicity', 'freebase.type_profile.strict_included_types', 'Topic'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'R. Kelly'), ('Lil Wayne', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Baby', 'music.recording.song', 'Baby'), ('Musician', 'common.topic.subjects', 'Steve Blevins'), ('1.FM Top 40', 'broadcast.content.artist', 'OneRepublic'), ('Record producer', 'people.profession.specialization_of', 'Music executive'), ('Sunshine Radio', 'broadcast.content.broadcast', 'Sunshine Radio - 140kbps Stream'), ('DMX', 'broadcast.artist.content', '1Club.FM: Power'), ('J. Holiday', 'music.artist.label', 'The Island Def Jam Music Group'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw444'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty and a Beat (Remixes)', 'music.album.featured_artists', 'Nicki Minaj'), ('Beyoncé Knowles', 'broadcast.artist.content', 'PowerHitz'), ('m.0gxnp0c', 'base.popstra.dated.participant', 'Caitlin Beadles'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Pop music'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Brad Haugen'), ('1.FM Top 40', 'broadcast.content.artist', 'Enrique Iglesias'), ('New Kids on the Block', 'common.topic.notable_types', 'Musical Artist'), ('FLOW 103', 'broadcast.content.artist', 'Ciara'), ('m.0_w3zrs', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Height'), ('Hot Wired Radio', 'broadcast.content.artist', 'Nickelback'), ('Alanis Morissette', 'people.person.profession', 'Actor'), ('Tyga', 'music.artist.genre', 'Contemporary R&B'), ('Justin Timberlake', 'people.person.languages', 'English Language'), ('Michael Jackson', 'music.artist.genre', 'Rock music'), ('Redfoo', 'music.featured_artist.albums', 'Live My Life'), ('1.FM Top 40', 'broadcast.content.genre', 'Hip hop music'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98_q'), ('Kuk Harrell', 'people.person.profession', 'Record producer'), ('Barry Weiss', 'music.artist.label', 'The Island Def Jam Music Group'), ('Island Records', 'common.topic.notable_types', 'Record label'), ('Frank Ocean', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ch!pz'), (\"Destiny's Child\", 'music.artist.genre', 'Dance-pop'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Redfoo', 'music.featured_artist.recordings', 'Little Bird'), ('J. Holiday', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_73g3'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kelis'), ('Akon', 'music.artist.genre', 'Contemporary R&B'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Contemporary R&B', 'broadcast.genre.content', '181-beat'), ('Usher', 'people.person.profession', 'Actor'), ('Everlast', 'people.person.profession', 'Singer'), ('Estelle', 'people.person.profession', 'Model'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Rock music'), ('LeAnn Rimes', 'broadcast.artist.content', '1.FM Top 40'), ('That Should Be Me', 'award.award_winning_work.awards_won', 'm.0njvs9s'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhtjj'), ('1Club.FM: V101', 'broadcast.content.artist', 'Janet Jackson'), ('Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Khalil', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('G (USA)', 'film.content_rating.country', 'United States of America'), ('m.0njwb81', 'award.award_honor.ceremony', '2010 MuchMusic Video Awards'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.song', 'Beauty And A Beat'), ('6 Tre G', 'music.artist.label', 'The Island Def Jam Music Group'), ('Shaun Cassidy', 'people.person.gender', 'Male'), ('m.0sgkw_d', 'award.award_honor.award', \"Kids' Choice Award for Biggest Slime Of The Night\"), ('Live My Life', 'music.composition.composer', 'JC Sindress'), ('Pray', 'music.composition.composer', 'Nasri'), ('Kelis', 'people.person.nationality', 'United States of America'), ('Trick Daddy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3bl'), ('1Club.FM: Power', 'broadcast.content.artist', 'Aaliyah'), ('m.0z1ndbl', 'award.award_nomination.nominated_for', 'justinbieber'), ('Justin Bieber', 'celebrities.celebrity.sexual_relationships', 'm.0d33gyj'), ('Christina Aguilera', 'music.artist.genre', 'Contemporary R&B'), ('Eenie Meenie', 'common.topic.article', 'm.0bbz4pf'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulSmoothJazz.com'), ('6 Tre G', 'common.topic.notable_types', 'Musical Artist'), ('m.0z8s562', 'award.award_honor.ceremony', 'NME Awards 2011'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0z0tgz6', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0gxnnwc', 'people.sibling_relationship.sibling', 'Jazmyn Bieber'), ('Benny Blanco', 'music.producer.releases_produced', 'My World'), ('Miley Cyrus', 'music.artist.genre', 'Dance music'), ('Ashlee Simpson', 'people.person.profession', 'Dancer'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Athan Grace', 'people.person.nationality', 'United States of America'), ('m.0wjgrzc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Boyfriend (Dada Life remix)', 'common.topic.notable_types', 'Musical Recording'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('1.FM Top 40', 'broadcast.content.artist', 'Javier Colon'), ('Jennifer Lopez', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0t4syfh', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.composer', 'Justin Bieber'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Justin Bieber', 'music.composer.compositions', 'Live My Life'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0vb6hhj'), ('Beauty And A Beat', 'music.composition.composer', 'Savan Kotecha'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Marvin Isley'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0zbf_4g'), ('m.0yr9tjb', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Roller Coaster', 'common.topic.notable_for', 'g.1yn5d20x3'), ('Justin Bieber', 'music.artist.album', 'Believe'), ('Twista', 'music.artist.origin', 'Chicago'), ('1.FM Top 40', 'broadcast.content.artist', 'Keane'), ('Gender', 'rdf-schema#range', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Jillian Twigger Moul'), ('Never Say Never: The Remixes', 'music.album.release_type', 'Album'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Snoop Dogg'), ('m.0njvvj_', 'award.award_honor.award', 'Juno Fan Choice Award'), (\"Destiny's Child\", 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cp'), ('HitzRadio.com', 'broadcast.content.artist', 'DMX'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Rihanna'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk0mt'), ('Somebody to Love (remix)', 'music.album.releases', 'Somebody to Love (remix)'), ('Shaffer Smith', 'people.person.profession', 'Actor'), ('First Dance', 'music.composition.composer', 'Jesse Wilson'), ('Sean Combs', 'people.person.profession', 'Singer'), ('Reed Smoot', 'people.person.nationality', 'United States of America'), ('Kings of Leon', 'influence.influence_node.influenced_by', 'Radiohead'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.featured_artists', 'Big Sean'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gj0bj6'), ('Rudolph Valentino', 'people.person.profession', 'Film Producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jagged Edge'), ('The Island Def Jam Music Group', 'common.topic.notable_for', 'g.125b9qymk'), ('Daniel Bedingfield', 'people.person.gender', 'Male'), ('Ginuwine', 'people.person.profession', 'Dancer'), ('Michael Jackson', 'broadcast.artist.content', 'radioIO Classic RNB'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.featured_artists', 'Big Sean'), ('Confident', 'music.album.primary_release', 'Confident'), ('Jordin Sparks', 'people.person.profession', 'Artist'), ('Brandy Norwood', 'people.person.profession', 'Dancer'), ('Justin Timberlake', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Kid Cudi', 'people.person.profession', 'Singer-songwriter'), ('Mariah Carey', 'music.artist.genre', 'Pop music'), ('Heartbreaker', 'music.album.artist', 'Justin Bieber'), ('Place of birth', 'rdf-schema#range', 'Location'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Cory Gunz', 'music.artist.label', 'The Island Def Jam Music Group'), ('School Boy Records', 'music.record_label.artist', 'Rixton'), ('Zendaya: Behind the Scenes', 'film.film.personal_appearances', 'm.0w3gbtv'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Shaffer Smith'), ('HitzRadio.com', 'broadcast.content.artist', '112'), ('justinbieber', 'common.topic.notable_for', 'g.1q3sdnv9d'), ('Love Me', 'music.album.release_type', 'Single'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.distributors', 'm.0hvlt03'), ('City/Town/Village', 'freebase.type_profile.strict_included_types', 'Location'), ('Kanye West', 'broadcast.artist.content', 'Smoothbeats'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101ftp1', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Hip hop music', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('My Worlds', 'music.album.releases', 'My Worlds'), ('2011 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw444'), ('m.0101fv8j', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Jason Mraz', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('m.0gbm3d3', 'film.personal_film_appearance.person', 'Ludacris'), ('1Club.FM: Mix 106', 'common.topic.notable_types', 'Broadcast Content'), ('R. Kelly', 'people.person.profession', 'Music executive'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Nicki Minaj', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Right Here (featuring Drake)', 'music.recording.artist', 'Drake'), ('Live My Life', 'music.album.featured_artists', 'Redfoo'), ('m.0j1t2xs', 'base.firsts.first_achievement.category', 'Canadian'), ('m.0d_hbgr', 'common.webpage.resource', 'Justin Bieber Lyrics'), ('Michael Jackson', 'broadcast.artist.content', '1Club.FM: V101'), ('Never Let You Go', 'common.topic.notable_for', 'g.12h312yf9'), ('World music', 'music.genre.subgenre', 'Reggae'), ('Diplo', 'common.topic.notable_types', 'Musical Artist'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Sonique'), (\"Justin Bieber's Believe\", 'film.film.music', 'Nathan Lanier'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'William Orbit'), ('Pras', 'music.artist.genre', 'Reggae'), ('m.0z85n_1', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Timbaland', 'music.artist.genre', 'Contemporary R&B'), ('m.0ng_j6d', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Singer', 'people.profession.specializations', 'Classical Vocalist'), ('Snoop Dogg', 'people.person.profession', 'Singer-songwriter'), ('HitzRadio.com', 'broadcast.content.artist', 'Three Days Grace'), ('m.0gh0fdt', 'tv.tv_guest_role.episodes_appeared_in', 'Series 1, Show 14'), ('Nicki Minaj', 'music.artist.genre', 'Dance-pop'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'BWO'), ('m.0yr9tjb', 'award.award_nomination.ceremony', '2010 Young Hollywood Awards'), ('Mason Levy', 'people.person.gender', 'Male'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftmz'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Nina Sky'), ('Mason Levy', 'people.person.nationality', 'United States of America'), ('Wait for a Minute', 'music.composition.composer', 'Maejor'), ('Rodney Jerkins', 'music.composer.compositions', 'As Long as You Love Me'), ('Never Let You Go', 'common.topic.notable_types', 'Musical Album'), ('Live My Life', 'common.topic.notable_types', 'Composition'), ('Justin Timberlake', 'broadcast.artist.content', 'HitzRadio.com'), ('Whitney Houston', 'people.person.nationality', 'United States of America'), ('Dance music', 'broadcast.genre.content', 'JellyRadio.com'), ('m.0101fsz2', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Ellen DeGeneres'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Rock music'), ('Chingy', 'music.artist.genre', 'Dance-pop'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Black Eyed Peas', 'music.artist.genre', 'Contemporary R&B'), ('m.0101ftt8', 'film.film_crew_gig.crewmember', 'Kiyomi Hara'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Gavin DeGraw'), ('m.0v30srx', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Iggy Azalea', 'music.artist.label', 'The Island Def Jam Music Group'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('Composition', 'freebase.type_profile.strict_included_types', 'Topic'), ('Max Martin', 'music.artist.label', 'Island Records'), ('m.09y886k', 'common.webpage.topic', 'Teen idol'), ('Concert tour', 'freebase.type_hints.included_types', 'Topic'), ('Demi Lovato', 'people.person.profession', 'Dancer'), ('Never Say Never: The Remixes', 'common.topic.image', '518hQytx0BL.jpg'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('m.0101fvl8', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Justin Timberlake', 'broadcast.artist.content', '181-beat'), ('#thatPOWER', 'common.topic.notable_types', 'Canonical Version'), ('Jordan Pruitt', 'people.person.profession', 'Singer-songwriter'), ('m.0njdns_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Lil Wayne', 'people.person.profession', 'Actor'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Selena Gomez', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsc'), ('m.09wjs_l', 'common.webpage.topic', 'Record producer'), ('Bryan-Michael Cox', 'people.person.nationality', 'United States of America'), ('Sean Kingston', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3cp'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Gwen Stefani'), ('Lady Gaga', 'music.artist.genre', 'Electronic dance music'), ('Juvenile', 'people.person.profession', 'Musician'), ('Eenie Meenie', 'common.topic.notable_for', 'g.12h2y4ysq'), ('Chicago', 'location.hud_county_place.place', 'Chicago'), ('m.0njw59_', 'award.award_honor.award', 'MTV Europe Music Award for Best Pop'), ('Shaggy', 'music.artist.genre', 'Reggae'), ('Confident', 'music.album.featured_artists', 'Chance the Rapper'), ('m.0101ft35', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Demi Lovato', 'music.artist.genre', 'Synthpop'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'common.topic.notable_types', 'Broadcast Content'), ('Red Hot Chili Peppers', 'music.artist.genre', 'Rock music'), ('1.FM Top 40', 'broadcast.content.artist', 'Anastacia'), ('Right Here', 'music.recording.song', 'Right Here'), ('Gavin DeGraw', 'people.person.profession', 'Singer-songwriter'), ('m.0101ft0d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Madonna', 'broadcast.artist.content', '.977 The Hits Channel'), ('Donna Summer', 'music.artist.genre', 'Contemporary R&B'), ('2010 Young Hollywood Awards', 'award.award_ceremony.nominees', 'm.0yr9tjb'), ('Akon', 'people.person.languages', 'English Language'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ian Oliver'), ('Blu Cantrell', 'people.person.profession', 'Singer'), ('Recovery', 'common.topic.notable_for', 'g.1yg4dl30h'), ('m.0nfnx_3', 'film.personal_film_appearance.film', 'Bad 25'), ('Teen idol', 'base.icons.icon_genre.icons', 'Neil Sedaka'), ('Book', 'freebase.type_profile.strict_included_types', 'Topic'), ('Rita Ora', 'music.artist.genre', 'Contemporary R&B'), ('181-beat', 'broadcast.content.artist', 'Rihanna'), ('Jordan Pruitt', 'people.person.profession', 'Singer'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Mason Levy', 'music.artist.genre', 'Pop music'), ('m.0njw257', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Sheryl Crow', 'people.person.profession', 'Musician'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nasri', 'people.person.gender', 'Male'), (\"Destiny's Child\", 'broadcast.artist.content', 'HitzRadio.com'), ('Juno Awards of 2013', 'award.award_ceremony.awards_presented', 'm.0t4s_bn'), ('Mariah Carey', 'broadcast.artist.content', '1Club.FM: V101'), ('Rihanna', 'broadcast.artist.content', 'radioIO Todays RNB'), ('m.0101fszb', 'film.personal_film_appearance.person', 'Usher'), ('Jordan Francis', 'people.person.nationality', 'Canada'), ('Lolly', 'music.single.versions', 'Lolly'), ('Fabolous', 'broadcast.artist.content', '181-beat'), ('Mannie Fresh', 'people.person.profession', 'Musician'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Contemporary R&B'), ('Hot Wired Radio', 'broadcast.content.artist', 'Zac Brown Band'), ('Person', 'type.type.properties', 'Children'), ('Justin Bieber', 'music.composer.compositions', 'Die in Your Arms'), ('Britney Spears', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Madonna', 'people.person.profession', 'Dancer'), ('Juvenile', 'people.person.nationality', 'United States of America'), ('HitzRadio.com', 'broadcast.content.artist', 'Ginuwine'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('1Club.FM: V101', 'broadcast.content.artist', 'R. Kelly'), ('Britney Spears', 'music.artist.genre', 'Contemporary R&B'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Rock music'), ('Dr. Dre', 'people.person.profession', 'Musician'), ('Khalil', 'freebase.valuenotation.has_value', 'Parents'), ('Bigger', 'music.composition.lyricist', 'Frank Ocean'), ('Teen idol', 'base.icons.icon_genre.icons', 'Leonardo DiCaprio'), ('Will Smith', 'people.person.profession', 'Film Producer'), ('Juelz Santana', 'people.person.profession', 'Actor'), ('Mann', 'music.artist.genre', 'Hip hop music'), ('m.0gwhmhm', 'award.award_honor.ceremony', 'Juno Awards of 2011'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92g'), ('Baby', 'music.composition.recordings', 'Baby'), ('Justin Bieber', 'music.composer.compositions', 'Never Say Never'), ('Yves Bole', 'people.person.nationality', 'Italy'), ('1.FM Top 40', 'broadcast.content.artist', 'LeAnn Rimes'), ('181-beat', 'broadcast.content.artist', 'Baby Bash'), ('School Boy Records', 'music.record_label.artist', 'Asher Roth'), ('Ludacris', 'music.featured_artist.albums', 'All Around the World'), ('JoJo', 'music.artist.genre', 'Teen pop'), ('FLOW 103', 'broadcast.content.artist', 'The Pussycat Dolls'), ('m.0101fsyr', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Gavin DeGraw', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0_srv2b'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Snoop Dogg', 'people.person.profession', 'Actor'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Ashlee Simpson', 'people.person.profession', 'Actor'), ('m.0gbmnsy', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Lady Antebellum', 'broadcast.artist.content', 'radioIO Todays POP'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0_grqb8'), ('m.0pcr2qn', 'tv.tv_guest_role.character', \"Pizzi's Best Friend\"), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pray', 'music.album.primary_release', 'Pray'), ('HitzRadio.com', 'broadcast.content.artist', 'M.I.A.'), ('m.0ng_vkd', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Chris Brown', 'people.person.profession', 'Record producer'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Benny Blanco', 'music.composer.compositions', 'Eenie Meenie'), ('All That Matters', 'music.album.release_type', 'Single'), ('Tyga', 'award.award_nominee.award_nominations', 'm.0_xlw5f'), ('Snoop Dogg', 'broadcast.artist.content', 'Smoothbeats'), ('1.FM Top 40', 'broadcast.content.artist', 'Beverley Knight'), ('London', 'location.location.containedby', 'Area codes 519 and 226'), ('Cris Cab', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber', 'music.composer.compositions', 'Bigger'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('Thought of You', 'music.single.versions', 'Thought Of You'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'The All-American Rejects'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsc'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sean Paul'), ('Sean Combs', 'music.artist.genre', 'Synthpop'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot instrumental)'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Cascada'), ('1Club.FM: V101', 'broadcast.content.artist', 'Michael Jackson'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Rihanna', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('#thatPOWER', 'common.topic.notable_for', 'g.11b5lrbn8m'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvjm'), (\"Nickelodeon Mexico Kids' Choice Award for Favorite Song\", 'award.award_category.winners', 'm.0yrjkl1'), ('Beauty and a Beat', 'music.album.compositions', 'Beauty And A Beat'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordan Pruitt', 'music.artist.genre', 'Teen pop'), (\"Bill O'Dowd\", 'people.person.gender', 'Male'), ('Hot Wired Radio', 'common.topic.article', 'm.03hrz8s'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Sean Kingston', 'people.person.gender', 'Male'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('My World 2.0', 'music.album.genre', 'Rhythm and blues'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Alicia Keys', 'people.person.gender', 'Female'), ('Akon', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Hold Tight', 'music.recording.artist', 'Justin Bieber'), ('PYD', 'music.album.artist', 'Justin Bieber'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0n1ykxp', 'award.award_honor.award_winner', 'Ludacris'), ('Kuk Harrell', 'music.composer.compositions', 'Never Say Never'), ('Canada', 'location.country.administrative_divisions', 'Ontario'), ('radioIO Todays POP', 'broadcast.content.location', 'Tampa'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjkl1'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ice Cube'), ('Favorite Girl', 'music.album.primary_release', 'Favorite Girl'), ('Chef Tone', 'music.artist.genre', 'Contemporary R&B'), ('Beauty And A Beat', 'common.topic.notable_types', 'Composition'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.featured_artists', 'Big Sean'), ('m.0tjyljn', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('RedOne', 'music.artist.genre', 'Hip hop music'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0yrhrwc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0t_l7mp'), ('All Around The World', 'music.composition.place_of_first_performance', 'United States of America'), ('All Around The World', 'music.composition.recordings', 'All Around The World (featuring Ludacris)'), ('Right Here (featuring Drake)', 'music.recording.song', 'Right Here'), ('Christina Aguilera', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'freebase.valuenotation.has_no_value', 'Weblink'), ('m.0gbm3cx', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Beautiful', 'music.recording.featured_artists', 'Justin Bieber'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs16'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.artist', 'Justin Bieber'), ('Cory Gunz', 'people.person.profession', 'Singer'), ('Katy Perry', 'music.artist.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Ricky Nelson'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Classic hits'), ('Justin Bieber', 'celebrities.celebrity.legal_entanglements', 'm.0_cz5l3'), ('Kelly Clarkson', 'people.person.gender', 'Female'), ('PYD', 'music.album.releases', 'PYD'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Confident', 'music.composition.composer', 'Soundz'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Chris Brown'), ('2012 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0jvgmxc'), ('Akon', 'people.person.profession', 'Singer'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Children'), ('Change Me', 'common.topic.notable_for', 'g.1yn5bkttb'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Khalil'), ('Little Bird', 'music.recording.tracks', 'Little Bird'), ('Justin Bieber', 'music.composer.compositions', 'All Around The World'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ramon Zerano'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Pop music'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaffer Smith', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'common.topic.image', 'beirutnights.png'), ('Mariah Carey', 'people.person.profession', 'Model'), ('C1', 'people.person.gender', 'Male'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Miley Cyrus', 'people.person.profession', 'Actor'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j219nq'), ('PowerHitz', 'broadcast.content.genre', 'Rap music'), ('Hikaru Utada', 'people.person.profession', 'Record producer'), ('#thatPOWER', 'music.single.versions', '#thatPower'), ('LeAnn Rimes', 'people.person.profession', 'Singer'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Tonic'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Fabolous'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Snoop Dogg', 'music.artist.genre', 'Hip hop music'), ('Katy Perry: Part of Me', 'film.film.genre', 'Documentary film'), ('Will i Am', 'music.artist.genre', 'Dance-pop'), ('Beyoncé Knowles', 'music.artist.genre', 'Pop music'), ('Eminem', 'people.person.gender', 'Male'), ('Place of birth', 'rdf-schema#domain', 'Person'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z1jn32'), ('m.0_vmmj6', 'award.award_nomination.award_nominee', 'Nicki Minaj'), ('1.FM Top 40', 'broadcast.content.artist', '112'), ('Reed Smoot', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Live My Life', 'music.album.album_content_type', 'Studio album'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Tupac Shakur', 'broadcast.artist.content', 'FLOW 103'), (\"Dick Clark's Primetime New Year's Rockin' Eve 2013\", 'film.film.personal_appearances', 'm.0pcnqnb'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Italy', 'location.country.languages_spoken', 'Italian Language'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten radio)'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c920'), ('m.0njhx1b', 'award.award_honor.award', 'Billboard Music Award for Top Streaming Artist'), ('1Club.FM: Power', 'broadcast.content.artist', 'M.I.A.'), ('Believe Acoustic', 'music.album.artist', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fall Out Boy'), ('Shaffer Smith', 'broadcast.artist.content', 'JellyRadio.com'), ('Bad Day', 'music.composition.composer', 'James Giannos'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Never Say Never', 'common.topic.notable_types', 'Composition'), ('m.0v90p2b', 'award.award_honor.award_winner', 'Justin Bieber'), ('Record producer', 'freebase.equivalent_topic.equivalent_type', 'Record Producer'), ('Mason Levy', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0p85jpp'), ('Confident', 'music.composition.recordings', 'Confident'), ('Contemporary R&B', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), ('Lolly', 'common.topic.notable_for', 'g.1yfp2v185'), ('m.0z84kwx', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.featured_artist.recordings', 'Where Are Ü Now'), ('Teen idol', 'base.icons.icon_genre.icons', 'Ashley Tisdale'), ('m.0_vmmj6', 'award.award_nomination.ceremony', '2013 Radio Disney Music Awards'), ('m.0z2dr9y', 'award.award_nomination.nominated_for', 'justinbieber'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('m.0p85jpp', 'film.personal_film_appearance.film', 'Katy Perry: Part of Me'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Enrique Iglesias', 'broadcast.artist.content', '1.FM Top 40'), ('m.0gbm3fr', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Chef Tone', 'music.composer.compositions', 'Heartbreaker'), ('Bad Day', 'music.composition.recordings', 'Bad Day'), ('Bad Day', 'music.composition.composer', 'Chris Jasper'), ('Ray J', 'music.artist.genre', 'Hip hop music'), ('American Music Award for Favorite Pop/Rock Album', 'award.award_category.winners', 'm.0njgyk4'), ('U Smile', 'music.album.artist', 'Justin Bieber'), ('Judy Garland', 'people.person.profession', 'Singer'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('m.09y8cm9', 'common.webpage.topic', 'Teen idol'), ('Kanye West', 'broadcast.artist.content', 'Hot 97.7'), ('R. Kelly', 'music.featured_artist.albums', 'PYD'), ('Usher', 'broadcast.artist.content', 'WildFMRadio.com'), ('Never Say Never (acoustic)', 'music.recording.canonical_version', 'Never Say Never'), ('Snoop Dogg', 'broadcast.artist.content', 'JellyRadio.com'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Ethnicity', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Ginuwine', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Big Sean', 'people.person.profession', 'Singer'), ('Justin Bieber', 'people.person.ethnicity', 'Canadian'), ('Miley Cyrus', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Hot 108 Jamz', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0kyjls4'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mario Winans'), ('2012 Teen Choice Awards', 'time.event.people_involved', 'Demi Lovato'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pattie Mallette', 'common.topic.article', 'm.0n40czg'), ('Brandy Norwood', 'music.artist.genre', 'Contemporary R&B'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Shaffer Smith'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mark Morrison'), ('HitzRadio.com', 'broadcast.content.artist', 'Gavin DeGraw'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Place of birth'), ('Pattie Mallette', 'people.person.languages', 'g.11byb5qj3x'), ('m.0njhvsq', 'freebase.valuenotation.has_no_value', 'Winning work'), ('This is Justin Bieber', 'film.film.genre', 'Documentary film'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Ronald Isley'), ('Madonna', 'music.artist.genre', 'Electronic music'), ('Nelly Furtado', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Timberlake', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Wait For a Minute', 'music.album.release_type', 'Single'), ('L.A. Reid', 'common.topic.notable_types', 'Record Producer'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO RNB Mix'), ('As Long as You Love Me (album version)', 'music.recording.featured_artists', 'Big Sean'), ('HitzRadio.com', 'broadcast.content.artist', 'Ciara'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty And A Beat', 'common.topic.article', 'm.0k2b595'), ('Fergie', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctps1'), ('Sia Furler', 'music.artist.genre', 'Pop music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Sir Mix-a-Lot'), ('Teyana', 'music.artist.genre', 'Pop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Jimmy Eat World'), ('Under the Mistletoe', 'music.album.genre', 'Holiday'), ('Benny Blanco', 'music.artist.genre', 'Contemporary R&B'), ('Teen idol', 'common.topic.webpage', 'm.09y886k'), ('Johntá Austin', 'music.artist.genre', 'Hip hop music'), ('m.0y803nt', 'award.award_honor.ceremony', '2011 Brit Awards'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty And A Beat', 'music.composition.recordings', 'Beautiful and the Beat'), ('m.0vp7m_x', 'music.recording_contribution.contributor', 'Raekwon'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lady Antebellum'), ('m.0z340zt', 'award.award_honor.award_winner', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.rating', 'PG (USA)'), ('Baby', 'award.award_winning_work.awards_won', 'm.0yrjkl1'), ('David Cassidy', 'base.icons.icon.icon_genre', 'Teen idol'), ('School Boy Records', 'music.record_label.artist', 'Psy'), ('181-beat', 'common.topic.notable_types', 'Broadcast Content'), ('Yves Bole', 'freebase.valuenotation.is_reviewed', 'Notable types'), ('Kylie Minogue', 'music.artist.genre', 'Electronic music'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'AEG Live'), ('m.043082k', 'common.webpage.category', 'Topic Webpage'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Gwen Stefani', 'people.person.profession', 'Record producer'), ('August Rigo', 'music.artist.label', 'The Island Def Jam Music Group'), ('Leonardo DiCaprio', 'people.person.nationality', 'United States of America'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0gbmnt3', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Pop music'), ('Contemporary R&B', 'broadcast.genre.content', 'Dif.'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('Jessica Simpson', 'music.artist.genre', 'Contemporary R&B'), ('Miami', 'base.biblioness.bibs_location.country', 'United States of America'), ('Ricky Nelson', 'people.person.profession', 'Actor'), ('Recovery', 'common.topic.notable_types', 'Musical Album'), ('m.0z87597', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0vp800w', 'music.recording_contribution.contributor', 'Ludacris'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('All That Matters', 'common.topic.notable_types', 'Composition'), ('All Around The World (featuring Ludacris)', 'music.recording.releases', 'Believe'), ('m.012nv5h3', 'people.place_lived.person', 'Yves Bole'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Year'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Gender'), ('London', 'base.wikipedia_infobox.settlement.area_code', 'Area codes 519 and 226'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Fabolous'), ('School Boy Records', 'music.record_label.artist', 'Martin Garrix'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'September'), ('Terius Nash', 'people.person.profession', 'Singer-songwriter'), ('Katy Perry', 'music.artist.genre', 'Dance music'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.personal_appearances', 'm.0j_tkcq'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k21'), ('Michael Jackson', 'people.person.nationality', 'United States of America'), ('Lolly', 'music.recording.song', 'Lolly'), ('Justin Bieber', 'music.artist.album', 'All Around the World'), ('m.0101fv9w', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Spouse (or domestic partner)', 'type.property.master_property', 'Spouse'), ('m.0101ft5f', 'film.personal_film_appearance.person', 'Kuk Harrell'), ('Tupac Shakur', 'people.person.languages', 'English Language'), ('HitzRadio.com', 'broadcast.content.artist', 'Fabolous'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Height'), ('Timbaland', 'people.person.profession', 'Actor'), ('1Club.FM: Power', 'broadcast.content.artist', 'Trick Daddy'), ('JellyRadio.com', 'broadcast.content.genre', 'Pop music'), ('Ciara', 'people.person.profession', 'Actor'), ('Jaden Smith', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3d_'), ('m.0yqflhy', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('Nathan Lanier', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0j_tkcq'), ('Kanye West', 'music.artist.genre', 'Contemporary R&B'), ('Somebody to Love (J Stax remix)', 'common.topic.notable_types', 'Musical Recording'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Singer'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0yqfny6', 'award.award_honor.award_winner', 'Justin Bieber'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Janet Jackson', 'music.artist.genre', 'Pop music'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0y_g556'), ('Lil Jon', 'broadcast.artist.content', '1Club.FM: Power'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Paula DeAnda'), ('Teen idol', 'common.topic.webpage', 'm.09y8bsk'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Height'), ('Lil Wayne', 'people.person.nationality', 'United States of America'), ('Jordin Sparks', 'people.person.profession', 'Musician'), ('Jane Lipsitz', 'freebase.valuenotation.has_value', 'Parents'), ('Never Say Never: The Remixes', 'music.album.artist', 'Justin Bieber'), ('Live My Life', 'music.single.versions', 'Little Bird'), ('Akon', 'broadcast.artist.content', '.977 The Hits Channel'), ('PYD', 'common.topic.notable_types', 'Musical Recording'), ('Hot Wired Radio', 'broadcast.content.artist', 'The Fray'), ('P!nk', 'music.artist.genre', 'Dance music'), ('Justin Timberlake', 'music.artist.genre', 'Dance-pop'), ('Alicia Keys', 'people.person.profession', 'Singer-songwriter'), ('#Thatpower', 'music.recording.song', '#thatPower'), ('m.0zg2qjr', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('m.0njhxzc', 'award.award_honor.honored_for', 'My World 2.0'), ('m.0njvtth', 'award.award_honor.award', 'Juno Award for Pop Album of the Year'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('HitzRadio.com', 'broadcast.content.artist', 'Flyleaf'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Fergie'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Somebody to Love (J Stax remix)', 'music.recording.song', 'Somebody to Love'), ('Never Say Never: The Remixes', 'music.album.genre', 'Dance-pop'), ('Jason Mraz', 'music.artist.genre', 'Reggae'), ('Mariah Carey', 'people.person.profession', 'Film Producer'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0ywtfj6'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Biographical Documentaries'), ('All Around The World', 'common.topic.notable_types', 'Composition'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrk18w'), ('Tricky Stewart', 'freebase.valuenotation.has_value', 'Parents'), ('m.0y84ynj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Beauty and a Beast', 'music.recording.featured_artists', 'Nicki Minaj'), ('WildFMRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Ja Rule', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0njwb81', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105zs8k'), ('#thatPOWER', 'music.single.versions', '#Thatpower'), ('Dr. Dre', 'broadcast.artist.content', 'Smoothbeats'), ('m.0b48fj5', 'common.webpage.topic', 'Record producer'), ('Big Sean', 'music.composer.compositions', 'As Long as You Love Me'), ('Baby', 'music.composition.composer', 'Terius Nash'), ('Miley Cyrus', 'broadcast.artist.content', 'Hot Wired Radio'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Christina Aguilera', 'people.person.profession', 'Singer-songwriter'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Michael Jackson', 'influence.influence_node.influenced', 'Britney Spears'), ('Beyoncé Knowles', 'broadcast.artist.content', '.977 The Hits Channel'), ('Profession', 'type.property.expected_type', 'Profession'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkw8v'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.song', 'Beauty And A Beat'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Baby Bash'), ('Live My Life', 'music.single.versions', 'Live My Life (Party Rock remix)'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Children'), ('Mariah Carey', 'people.person.profession', 'Record producer'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien dubstep mix)'), ('Pop music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('All Around the World', 'music.album.release_type', 'Single'), ('Dancer', 'music.special_music_video_performance_type.special_music_video_performances', 'm.0z3vx9q'), ('Kelly Clarkson', 'people.person.nationality', 'United States of America'), ('Nelly', 'people.person.profession', 'Singer-songwriter'), ('NME Award for Worst Album', 'award.award_category.nominees', 'm.0z8t2dy'), ('m.0y_g42w', 'award.award_nomination.nominated_for', 'justinbieber'), ('Max Martin', 'music.artist.genre', 'Dance music'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Release type'), ('WildFMRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Hot Wired Radio', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('2010 Young Hollywood Awards', 'award.award_ceremony.awards_presented', 'm.0yr9c1k'), ('Whitney Houston', 'music.artist.genre', 'Dance-pop'), ('Anastacia', 'music.artist.label', 'The Island Def Jam Music Group'), ('Believe Acoustic', 'common.topic.notable_for', 'g.126tqpnyl'), ('Justin Bieber', 'common.topic.webpage', 'm.07053l5'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt8'), ('Gavin DeGraw', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Hip hop music', 'broadcast.genre.content', 'Hot 97.7'), ('m.0njhtjj', 'award.award_honor.award', 'Billboard Music Award for Top New Artist'), ('m.0y_g556', 'award.award_honor.ceremony', '4th Annual Shorty Awards'), ('1.FM Top 40', 'broadcast.content.artist', 'Van Halen'), ('Brandy Norwood', 'people.person.profession', 'Record producer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ferry Corsten'), ('2010 MuchMusic Video Awards', 'award.award_ceremony.awards_presented', 'm.0njwb81'), ('Chris Brown', 'people.person.gender', 'Male'), ('m.0ng_vkd', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Blu Cantrell', 'broadcast.artist.content', 'DeeGay'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'Boys Like Girls'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Mary J. Blige', 'broadcast.artist.content', '1Club.FM: V101'), ('m.0njhtjj', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kanye West', 'people.person.gender', 'Male'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.featured_artists', 'Big Sean'), ('Baby', 'music.composition.recordings', 'Baby'), ('Amerie', 'music.artist.genre', 'K-pop'), ('This is Justin Bieber', 'film.film.personal_appearances', 'm.0ng_j6d'), ('Baby', 'music.composition.language', 'English Language'), ('Singer', 'people.profession.specializations', 'Singer-songwriter'), ('Die in Your Arms', 'music.composition.composer', 'Thomas Lumpkins'), ('Urban contemporary', 'broadcast.genre.content', 'Smoothbeats'), ('Jason Mraz', 'music.artist.genre', 'Acoustic music'), ('Live My Life (Party Rock remix)', 'common.topic.notable_types', 'Musical Recording'), ('Jay-Z', 'broadcast.artist.content', 'Smoothbeats'), ('Keyshia Cole', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: V101', 'broadcast.content.artist', 'Alicia Keys'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('Duffy', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Alicia Keys', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0102z0vx'), ('Katy Perry', 'music.artist.label', 'Island Records'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0t_l7mp'), ('My World', 'common.topic.article', 'm.0805zjd'), ('Bryan Adams', 'music.artist.genre', 'Pop music'), ('Scooter Braun', 'music.artist.label', 'School Boy Records'), ('Boyfriend (Dada Life remix)', 'music.recording.canonical_version', 'Boyfriend'), ('Terius Nash', 'music.artist.genre', 'Contemporary R&B'), ('Fergie', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Diplo', 'freebase.valuenotation.has_value', 'Parents'), ('Sean Combs', 'broadcast.artist.content', '181-beat'), ('Runaway Love (remix)', 'music.recording.canonical_version', 'Runaway Love'), ('Lady Gaga', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('RedOne', 'music.artist.genre', 'Pop music'), ('Katy Perry', 'music.artist.genre', 'Dance-pop'), ('Juelz Santana', 'people.person.profession', 'Musician'), ('Rihanna', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Gwen Stefani', 'music.artist.genre', 'Electronic music'), ('Dance music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('Avery', 'music.artist.genre', 'Synthpop'), ('Will Smith', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Recovery', 'music.composition.composer', 'Craig David'), ('Geri Halliwell', 'music.artist.genre', 'Pop music'), ('Johntá Austin', 'people.person.profession', 'Record producer'), ('KooL CrAzE', 'music.artist.genre', 'Hip hop music'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Parents'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Simple Plan'), ('Person', 'freebase.type_hints.included_types', 'Topic'), (\"Pizzi's Best Friend\", 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr2qn'), ('Justin Bieber', 'music.artist.contribution', 'm.0vp7mrq'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Next to You', 'common.topic.notable_for', 'g.12lqg_0ly'), ('Ashley Tisdale', 'music.artist.genre', 'Dance-pop'), ('m.0ng_k21', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('Camagüey', 'location.location.time_zones', 'Eastern Time Zone'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Music'), ('m.0jztshx', 'film.performance.actor', 'Justin Bieber'), ('Contemporary R&B', 'broadcast.genre.content', 'Hot 97.7'), ('Chloe Bridges', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber: Never Say Never', 'common.topic.image', '6759bc3bDlbiBmi_1_l.jpg'), ('Ice Cube', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Baby', 'music.recording.canonical_version', 'Baby'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjmq'), ('World music', 'broadcast.genre.content', 'DeeGay'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0z340zt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Hot Wired Radio', 'broadcast.content.genre', 'Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gsj'), ('Justin Timberlake', 'people.person.profession', 'Film Producer'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0cq990t', 'common.webpage.resource', 'Justin Bieber Romania'), ('Tupac Shakur', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Usher'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Justin Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Hot Wired Radio', 'broadcast.content.genre', 'Hip hop music'), ('Pras', 'people.person.nationality', 'United States of America'), ('Believe', 'music.album.primary_release', 'Believe'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.composition.composer', 'Christina Milian'), ('Chris Brown', 'broadcast.artist.content', 'PowerHitz'), ('Nick Jonas', 'music.artist.genre', 'Contemporary R&B'), ('Sunshine Radio', 'broadcast.content.genre', 'International'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Children'), ('Chris Brown', 'music.artist.genre', 'Urban contemporary'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Miley Cyrus'), ('Beauty and a Beat', 'music.recording.artist', 'Nicki Minaj'), ('All Around the World', 'music.album.featured_artists', 'Ludacris'), ('Selena Gomez', 'music.artist.genre', 'Rock music'), ('Kylie Minogue', 'people.person.profession', 'Record producer'), ('m.0gbm3d3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Ellen DeGeneres', 'people.person.gender', 'Female'), ('Redfoo', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h523f'), ('All That Matters', 'music.composition.recordings', 'All That Matters (video)'), ('Never Say Never', 'music.album.featured_artists', 'Jaden Smith'), ('Thought Of You', 'common.topic.notable_types', 'Musical Recording'), ('50 Cent', 'broadcast.artist.content', 'WildFMRadio.com'), ('Shaggy', 'people.person.profession', 'Actor'), ('m.0z898w6', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('m.0gbm35z', 'film.film_cut.film_release_region', 'United States of America'), ('Vanessa Hudgens', 'music.artist.genre', 'Teen pop'), ('Eminem', 'people.person.profession', 'Actor'), ('Johntá Austin', 'people.person.gender', 'Male'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Bad Day', 'music.album.release_type', 'Single'), ('m.0z3tqqt', 'award.award_nomination.award', 'Shorty Award for Music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_j6d'), ('Teen Choice Award for Choice Music: Album Pop', 'award.award_category.nominees', 'm.0z8qqh5'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Love Never Felt So Good', 'common.topic.notable_types', 'Musical Album'), ('Yves Bole', 'influence.influence_node.influenced', 'iJustine'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0jvgmxc', 'award.award_honor.ceremony', '2012 Billboard Music Awards'), ('R. Kelly', 'music.artist.genre', 'Contemporary R&B'), ('Foreign Remix', 'music.recording.tracks', 'Foreign Remix'), ('30 Days in May', 'common.topic.notable_types', 'Film'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('BeirutNights.com Radio', 'broadcast.content.broadcast', 'BeirutNights.com Radio - 128kbps Stream'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Italy', 'location.country.official_language', 'Italian Language'), ('Clay Aiken', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0njdq6k', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0z8qx3w', 'award.award_honor.award', 'Teen Choice Award for Choice Music: Album Pop'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Dance-pop', 'common.topic.notable_types', 'Musical genre'), ('Under the Mistletoe', 'music.album.releases', 'Under the Mistletoe'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'September'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Musical Artist', 'type.type.expected_by', 'Artists'), ('Chris Brown', 'people.person.profession', 'Dancer'), ('m.0yrjkl1', 'award.award_honor.award', \"Nickelodeon Mexico Kids' Choice Award for Favorite Song\"), ('Beauty And A Beat', 'music.composition.composer', 'Zedd'), ('m.0njgyk4', 'award.award_honor.award_winner', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('m.0101ftl5', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Big Sean'), ('R. Kelly', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Canada', 'location.country.languages_spoken', 'English Language'), ('1.FM Top 40', 'broadcast.content.artist', 'Max Graham vs. Yes'), ('The Roots', 'broadcast.artist.content', '1Club.FM: V101'), ('Janet Jackson', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Adrienne Bailon', 'people.person.profession', 'Actor'), ('Teen idol', 'base.icons.icon_genre.icons', 'David Cassidy'), ('Foreign Remix', 'music.recording.featured_artists', 'Justin Bieber'), ('Musical Artist', 'freebase.type_profile.equivalent_topic', 'Musician'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0z1jn32', 'award.award_nomination.nominated_for', 'justinbieber'), ('School Gyrls', 'film.film.country', 'United States of America'), ('Ciara', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.artist', 'Justin Bieber'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO Todays POP'), ('Singer', 'common.topic.subject_of', 'Léo Ferré'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cg'), ('Sean Kingston', 'music.artist.album', 'Eenie Meenie'), ('Pray', 'music.composition.recordings', 'Pray'), ('m.0jvgmxc', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jazmyn Bieber', 'people.person.nationality', 'Canada'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('U Smile', 'common.topic.notable_for', 'g.12h2_3_z_'), ('Baby', 'music.recording.song', 'Baby'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('m.0t4syfh', 'award.award_nomination.ceremony', 'Juno Awards of 2012'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Willa Ford'), ('m.0j8z6tl', 'award.award_nomination.ceremony', '2012 Billboard Music Awards'), ('Teen idol', 'base.icons.icon_genre.icons', 'Josue Diaz'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Rudolph Valentino', 'people.person.profession', 'Dancer'), ('m.07053l5', 'common.webpage.category', 'Topic Webpage'), ('Where Are Ü Now', 'music.recording.artist', 'Jack Ü'), ('Khalil', 'music.artist.genre', 'Hip hop music'), ('Justin Bieber', 'music.artist.album', 'Under the Mistletoe'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.song', 'As Long as You Love Me'), ('Jeremy Bieber', 'people.person.profession', 'Musician'), ('m.0njhyh_', 'award.award_honor.honored_for', 'Baby'), ('PowerHitz', 'broadcast.content.artist', 'Usher'), ('Lady Gaga', 'music.artist.genre', 'Dance music'), ('radioIO Todays POP', 'broadcast.content.genre', 'Pop music'), ('m.0yrlqp1', 'education.education.student', 'Justin Bieber'), ('Rita Ora', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Christina Aguilera'), ('Alicia Keys', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.music', 'Deborah Lurie'), ('Santana', 'music.artist.genre', 'Rock music'), ('NME Award for Worst Album', 'award.award_category.winners', 'm.0z8s_wn'), ('Colbie Caillat', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'theater.theater_company.plays_produced', 'Def Poetry'), ('Jay-Z', 'people.person.profession', 'Actor'), ('Baby', 'music.album.releases', 'Baby'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Britney Spears'), ('William Orbit', 'common.topic.notable_types', 'Musical Artist'), ('Drake', 'people.person.profession', 'Singer'), ('Emphatic Radio.com!', 'common.topic.notable_for', 'g.1255_9wpq'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Cassie Ventura'), ('Radio Disney Music Award for Best Male Artist', 'award.award_category.winners', 'm.0y4tdml'), ('Justin Bieber', 'music.featured_artist.albums', '#thatPOWER'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'No Doubt'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Keyshia Cole', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'people.person.profession', 'Singer'), ('Kelis', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('iJustine', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('Lady Antebellum', 'music.artist.genre', 'Pop music'), ('Sir Nolan', 'people.person.nationality', 'United States of America'), ('Max Martin', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.single.versions', 'Boyfriend'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.has_value', 'Parents'), ('m.0z8s_wn', 'award.award_honor.award', 'NME Award for Worst Album'), ('Michael Jackson', 'influence.influence_node.influenced', 'Justin Timberlake'), ('School Gyrls', 'film.film.produced_by', 'L.A. Reid'), ('Nelly', 'broadcast.artist.content', 'Hot 108 Jamz'), ('1.FM Top 40', 'broadcast.content.artist', 'Delerium'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The BossHoss'), ('Teen idol', 'base.icons.icon_genre.icons', 'Annette Funicello'), ('Teen idol', 'common.topic.webpage', 'm.09wf9d1'), ('Jay-Z', 'broadcast.artist.content', 'HitzRadio.com'), ('Pattie Mallette', 'people.person.profession', 'Writer'), ('Michael Jackson', 'people.person.profession', 'Record producer'), (\"Justin Bieber's Believe\", 'film.film.country', 'United States of America'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Height'), ('Kelis', 'music.artist.genre', 'Dance music'), ('Twista', 'people.person.nationality', 'United States of America'), ('m.05sp405', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('Justin Timberlake', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('DMX', 'broadcast.artist.content', 'HitzRadio.com'), ('Demi Lovato', 'music.artist.genre', 'Contemporary R&B'), ('Gwen Stefani', 'music.artist.genre', 'Dance music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Frank Sinatra'), ('Heartbreaker', 'music.album.primary_release', 'Heartbreaker'), ('Jessie J', 'people.person.profession', 'Singer'), ('m.012nv3hv', 'business.employment_tenure.person', 'Yves Bole'), ('Janet Jackson', 'music.artist.genre', 'Rhythm and blues'), ('Pearl Jam', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('Snoop Dogg', 'people.person.languages', 'English Language'), ('m.0w5l5h1', 'education.education.institution', 'St. Michael Catholic Secondary School'), ('Elvis Presley', 'people.person.profession', 'Musician'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_73g3'), ('m.0y4tdml', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Nelly', 'people.person.profession', 'Record producer'), ('m.0njhx1b', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award category'), ('m.0101ftt8', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Kuk Harrell', 'music.artist.genre', 'Dance-pop'), ('Shaggy', 'common.topic.notable_types', 'Musical Artist'), ('Under the Mistletoe', 'music.album.release_type', 'Album'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Club Mix)'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Conjure One'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Profession'), ('#Thatpower', 'common.topic.notable_for', 'g.11b5m2tmhn'), ('Keyshia Cole', 'people.person.nationality', 'United States of America'), ('m.0zbf_4g', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Singer-songwriter', 'common.topic.subject_of', 'Bleona'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvc3'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4x'), ('Gavin DeGraw', 'broadcast.artist.content', '1.FM Top 40'), ('m.0_sxby0', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('As Long as You Love Me', 'music.album.artist', 'Justin Bieber'), ('1Club.FM: V101', 'broadcast.content.artist', 'Brandy Norwood'), ('1Club.FM: Power', 'broadcast.content.producer', '1Club.FM'), ('All That Matters', 'music.composition.composer', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'common.topic.notable_types', 'Musical Recording'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109j5xt'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('P!nk', 'music.artist.genre', 'Rhythm and blues'), ('Nicki Minaj', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('HitzRadio.com', 'broadcast.content.artist', 'Lady Gaga'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('First Dance', 'music.composition.composer', 'Dwight Reynolds'), ('m.0101ft1r', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Boyfriend', 'music.single.versions', 'Boyfriend (Dada Life remix)'), ('Pop music', 'music.genre.parent_genre', 'Rhythm and blues'), ('Beauty And A Beat', 'music.composition.composer', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jessie James Decker'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Montell Jordan', 'common.topic.notable_types', 'Musical Artist'), ('Foreign Remix', 'music.recording.tracks', 'Foreign Remix'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Blake Lewis'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0njhvsq'), (\"O'Kelly Isley, Jr.\", 'music.composer.compositions', 'Bad Day'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.05sp405'), ('Paul Anka', 'people.person.nationality', 'United States of America'), ('Nelly Furtado', 'music.artist.genre', 'Hip hop music'), ('Green Day', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0gbm3cg', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nathan Lanier', 'people.person.gender', 'Male'), ('Jessie J', 'music.artist.label', 'Island Records'), ('Caitlin Beadles', 'base.popstra.celebrity.breakup', 'm.0gxnp26'), ('Yves Bole', 'people.person.places_lived', 'm.012nv5h3'), ('P!nk', 'music.artist.genre', 'Pop music'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Dapo Torimiro', 'music.artist.genre', 'Rock music'), ('Madonna', 'influence.influence_node.influenced', 'Lady Gaga'), ('Donna Summer', 'people.person.gender', 'Female'), ('Dance music', 'music.genre.subgenre', 'K-pop'), ('m.0sxhgqj', 'award.award_nomination.ceremony', 'Juno Awards of 2013'), ('Britney Spears', 'music.artist.genre', 'Teen pop'), ('m.0j2189_', 'music.music_video_performance.music_video_character', 'Singer'), ('Favorite Girl', 'music.album.releases', 'Favorite Girl'), ('Zendaya: Behind the Scenes', 'common.topic.notable_types', 'Film'), ('Roller Coaster', 'music.album.artist', 'Justin Bieber'), ('Blackstreet', 'music.artist.genre', 'Hip hop music'), ('m.09wxbsr', 'common.webpage.topic', 'Record producer'), ('m.0njvtth', 'award.award_honor.honored_for', 'My World 2.0'), ('London', 'location.location.containedby', 'Canada'), ('Never Let You Go', 'music.composition.lyricist', 'Justin Bieber'), ('m.0nh04xx', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ernie Isley', 'people.person.gender', 'Male'), ('Urban contemporary', 'broadcast.genre.content', '181-thebox'), ('Kid Cudi', 'people.person.languages', 'English Language'), ('Usher', 'people.person.profession', 'Music executive'), ('1Club.FM: V101', 'broadcast.content.genre', 'Classic hits'), ('Nick Jonas', 'people.person.profession', 'Singer-songwriter'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Gollum & Yanny'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('Gavin DeGraw', 'common.topic.notable_types', 'Musical Artist'), ('m.0kyjls4', 'music.music_video_performance.music_video_character', 'Singer'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful and the Beat', 'common.topic.notable_types', 'Musical Recording'), ('Kid Cudi', 'people.person.gender', 'Male'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Lil Jon'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Lil Wayne'), ('1.FM Top 40', 'broadcast.content.artist', 'Britney Spears'), ('Twista', 'broadcast.artist.content', '1.FM Top 40'), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.episode', \"Boot Camp No. 3 / Judge's House No. 1\"), ('Jason Mraz', 'music.artist.genre', 'Indie rock'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ray J', 'music.artist.genre', 'Rhythm and blues'), ('Smoothbeats', 'common.topic.notable_types', 'Broadcast Content'), ('Dapo Torimiro', 'music.artist.genre', 'Pop music'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Jay-Z'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.episode', 'Episode 3'), ('Sunshine Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Juicy J', 'people.person.gender', 'Male'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Chloe Bridges'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Jason McCann', 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr2dh'), ('m.011m26pv', 'music.group_membership.role', 'Vocals'), ('Pray', 'music.composition.form', 'Song'), ('Yves Bole', 'base.musicpf.artist.genre', 'Singer-songwriter'), ('Janet Jackson', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('R. Kelly', 'people.person.profession', 'Singer'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Usher'), ('m.0gbm3bl', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('The Notorious B.I.G.', 'people.person.profession', 'Singer'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'King Ku$ha'), ('m.0yrjynf', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('Thought Of You', 'music.recording.artist', 'Justin Bieber'), ('m.0njdq6k', 'award.award_honor.award_winner', 'Justin Bieber'), ('Miami', 'location.hud_county_place.place', 'Miami'), ('Bad Day', 'common.topic.notable_for', 'g.1yg4drkzk'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bm2h'), ('Akon', 'music.artist.genre', 'Reggae'), ('m.0njhx1b', 'award.award_honor.award_winner', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', \"Nino D'Angelo\"), ('Madonna', 'people.person.languages', 'English Language'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Kanye West', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Shaffer Smith', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.written_by', 'Sarah Landman'), ('My Worlds', 'common.topic.notable_types', 'Musical Album'), ('Bigger', 'common.topic.notable_for', 'g.1z25545x7'), ('Rock music', 'common.topic.subject_of', 'Alan Motley'), ('PowerHitz', 'broadcast.content.artist', 'King Ku$ha'), ('Keyshia Cole', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr2qn'), ('iJustine', 'people.person.profession', 'Actor'), ('Shaffer Smith', 'music.artist.genre', 'Europop'), ('Toby Gad', 'music.artist.genre', 'Dance music'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Kevin Risto', 'music.lyricist.lyrics_written', 'Bigger'), ('Die in Your Arms', 'music.album.releases', 'Die in Your Arms'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Nelly Furtado', 'people.person.profession', 'Singer-songwriter'), ('m.0gbm3cp', 'film.personal_film_appearance.person', 'Sean Kingston'), ('Date of birth', 'type.property.expected_type', 'Date/Time'), ('HitzRadio.com', 'broadcast.content.artist', 'Christina Milian'), ('Dance music', 'broadcast.genre.content', '1.FM Top 40'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Big Sean', 'music.artist.track', 'As Long As You Love Me'), ('Jaden Smith', 'people.person.profession', 'Actor'), ('Duffy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Estelle'), ('Diplo', 'music.artist.genre', 'Electronic music'), ('Jayceon Terrell Taylor', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.artist', 'Sean Paul'), ('Confident', 'music.album.artist', 'Justin Bieber'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Men in Black 3', 'common.topic.notable_types', 'Award-Winning Work'), ('m.0yrjvlh', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('My Worlds: The Collection', 'common.topic.image', 'Justin-Bieber-My-Worlds-The-Collection.jpg'), ('Record producer', 'common.topic.webpage', 'm.09wxbsr'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011vfnlz'), ('Christina Milian', 'people.person.profession', 'Singer-songwriter'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('MTV Europe Music Award for Best World Stage Performance', 'award.award_category.winners', 'm.0z340zt'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0j8z6tl'), ('HitzRadio.com', 'broadcast.content.artist', 'Nickelback'), ('All Around the World', 'music.album.releases', 'All Around the World'), ('Never Let You Go', 'music.album.artist', 'Justin Bieber'), ('Chris Jasper', 'common.topic.notable_types', 'Musical Artist'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Craig David'), ('Nicki Minaj', 'music.artist.genre', 'Hip hop music'), ('Runaway Love (remix)', 'music.recording.featured_artists', 'Raekwon'), ('Justin Bieber', 'music.composer.compositions', 'Never Let You Go'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_70rd'), ('Jessie J', 'music.artist.genre', 'Dance-pop'), ('Nicki Minaj', 'music.artist.genre', 'Rhythm and blues'), ('m.0gbm3dn', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bad Day', 'common.topic.notable_types', 'Musical Recording'), ('Gwen Stefani', 'people.person.profession', 'Singer-songwriter'), ('Love Never Felt So Good', 'common.topic.notable_for', 'g.1s05bndft'), ('Big R Radio - The Hawk', 'broadcast.content.producer', 'Big R Radio Network'), ('Hot Wired Radio', 'broadcast.content.artist', 'Justin Timberlake'), ('Urban contemporary', 'broadcast.genre.content', 'Hot 97.7'), ('Jason Mraz', 'common.topic.notable_types', 'Musical Artist'), ('Ginuwine', 'broadcast.artist.content', 'HitzRadio.com'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0z898w6'), ('Lady Gaga', 'people.person.gender', 'Female'), ('Teen Choice Award for Choice Summer Music Star: Male', 'award.award_category.winners', 'm.0yrkgd6'), ('Right Here', 'common.topic.article', 'm.0qfpkqx'), ('Justin Bieber', 'music.featured_artist.recordings', 'Next to You'), ('Daniel Bedingfield', 'people.person.profession', 'Singer'), ('Fall Out Boy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('181-beat', 'broadcast.content.artist', 'Justin Timberlake'), ('As Long as You Love Me (acoustic version)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Award-Winning Work', 'type.type.expected_by', 'Winning work'), ('HitzRadio.com', 'broadcast.content.artist', 'Akon'), ('Blu Cantrell', 'people.person.profession', 'Musician'), ('Akon', 'music.artist.genre', 'Dance-pop'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hip hop music', 'broadcast.genre.content', '1Club.FM: Channel One'), ('Believe Tour', 'music.concert_tour.artist', 'Justin Bieber'), ('Anastacia', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'music.artist.album', 'Love Me'), ('As Long as You Love Me', 'music.composition.composer', 'Big Sean'), ('1.FM Top 40', 'broadcast.content.genre', 'Top 40'), ('Believe Acoustic', 'common.topic.article', 'm.0pb8twm'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Top 40'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('Don Henley', 'people.person.profession', 'Singer'), ('RedOne', 'music.artist.genre', 'Electronic dance music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Fray'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0tkqqgg'), ('Ronald Isley', 'freebase.valuenotation.has_value', 'Parents'), ('Actor', 'base.lightweight.profession.professions_similar', 'Model'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jayceon Terrell Taylor'), ('Ronald Isley', 'people.person.profession', 'Record producer'), ('Under the Mistletoe', 'common.topic.notable_types', 'Musical Album'), ('Justin Bieber', 'music.artist.genre', 'Contemporary R&B'), ('Amerie', 'people.person.profession', 'Model'), ('Lil Wayne', 'music.artist.genre', 'Hip hop music'), ('Jennifer Lopez', 'people.person.languages', 'English Language'), ('JoJo', 'music.artist.genre', 'Rhythm and blues'), ('Kylie Minogue', 'music.artist.genre', 'Rhythm and blues'), ('Boyfriend', 'common.topic.notable_types', 'Award-Winning Work'), ('Turn to You (Mother’s Day Dedication)', 'music.album.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Nelly Furtado'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc3_1'), ('Never Say Never: The Remixes', 'music.album.genre', 'Contemporary R&B'), ('Rob Thomas', 'people.person.nationality', 'United States of America'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftn8'), ('Beauty and a Beat (acoustic version)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('HitzRadio.com', 'broadcast.content.artist', 'Mariah Carey'), ('Trey Songz', 'people.person.profession', 'Singer-songwriter'), ('Terius Nash', 'music.artist.genre', 'Hip hop music'), ('All That Matters', 'common.topic.notable_for', 'g.1yfp37src'), ('m.0gxnp0c', 'base.popstra.dated.participant', 'Justin Bieber'), ('Beauty and a Beat', 'music.album.featured_artists', 'Nicki Minaj'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Monifah'), ('CSI: Crime Scene Investigation', 'tv.tv_program.country_of_origin', 'United States of America'), ('Justin Bieber Romania', 'common.resource.annotations', 'm.0cq990t'), ('Justin Bieber', 'base.schemastaging.person_extra.net_worth', 'm.0yqflhy'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Ricky Nelson', 'people.person.profession', 'Musician'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Official website'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Chingy'), ('Justin Bieber', 'music.artist.album', 'My Worlds: The Collection'), ('m.0z0tmyv', 'award.award_honor.ceremony', '4th Annual Shorty Awards'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Love Never Felt So Good', 'music.album.primary_release', 'Love Never Felt So Good'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Profession'), ('United States of America', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ludacris', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Children', 'type.property.master_property', 'Parents'), ('Annette Funicello', 'people.person.profession', 'Singer'), ('Ludacris', 'broadcast.artist.content', 'PowerHitz'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'music.composer.compositions', 'Wait for a Minute'), ('Selena Gomez', 'celebrities.celebrity.sexual_relationships', 'm.0gxnnzy'), ('Trick Daddy', 'broadcast.artist.content', '181-beat'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Teen idol', 'common.topic.webpage', 'm.09y8dts'), ('All That Matters (video)', 'common.topic.notable_types', 'Musical Recording'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Sean Combs'), ('Change Me', 'common.topic.notable_for', 'g.1yp3dqvz2'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Michael Jackson'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhxd_'), ('m.0sgkw_d', 'award.award_honor.award_winner', 'Justin Bieber'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Gender'), ('William Orbit', 'people.person.gender', 'Male'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Los Niños de Sara'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.article', 'm.0jwqwz0'), ('Somebody to Love (remix)', 'music.album.featured_artists', 'Usher'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Leona Lewis'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Will Smith'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Madonna'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Jessie J'), ('Justin Bieber', 'music.artist.album', 'Never Let You Go'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s_wn'), ('Teen pop', 'common.topic.notable_types', 'Musical genre'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Year'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Timbaland', 'music.group_member.artists_supported', 'Bleona'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Nelly', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('The Isley Brothers', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0sxhq3d'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Baby Bash'), ('Baby', 'music.composition.recordings', 'Baby'), ('Ludacris', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3d3'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjynf'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr2dh'), ('m.0102z0vx', 'award.award_honor.ceremony', 'Juno Awards of 2014'), ('m.0wjgqck', 'award.award_honor.award_winner', 'Justin Bieber'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien Luvstep mix)'), ('Never Say Never', 'common.topic.notable_for', 'g.1yl5lrh_1'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('Ashley Tisdale', 'music.artist.genre', 'Rhythm and blues'), ('Ellen DeGeneres', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft3d'), ('Pearl Jam', 'broadcast.artist.content', 'Hot Wired Radio'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Country of nationality', 'type.property.schema', 'Person'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kylie Minogue', 'music.artist.genre', 'Pop music'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronald Isley', 'people.person.profession', 'Singer'), ('Synthpop', 'music.genre.subgenre', 'Synthpop'), ('Chingy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('BeirutNights.com Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Jessica Simpson', 'people.person.gender', 'Female'), ('Singer-songwriter', 'people.profession.specialization_of', 'Singer'), ('m.0wjgrzc', 'award.award_honor.award_winner', 'Justin Bieber'), ('PowerHitz', 'broadcast.content.artist', 'Flo Rida'), ('HitzRadio.com', 'broadcast.content.artist', 'Buckcherry'), ('Record producer', 'common.topic.webpage', 'm.09wwfnk'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulClassics.com'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Alicia Keys'), ('Nelly Furtado', 'common.topic.notable_types', 'Musical Artist'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5kxx'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101ft2v', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Adrienne Bailon', 'music.artist.genre', 'Rhythm and blues'), ('Will i Am', 'people.person.profession', 'Dancer'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z1scxk'), ('Gavin DeGraw', 'broadcast.artist.content', 'HitzRadio.com'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Capone-N-Noreaga'), ('RedOne', 'music.artist.genre', 'Eurodance'), ('Hot Wired Radio', 'broadcast.content.artist', 'P!nk'), ('Roller Coaster', 'common.topic.notable_types', 'Musical Recording'), ('m.0101ftks', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Teen idol', 'common.topic.image', 'Rudolph Valentino'), ('m.0yrk4gn', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Savan Kotecha', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0gbm3d9', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Dan Cutforth', 'people.person.gender', 'Male'), ('Katy Perry', 'music.artist.genre', 'Teen pop'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Sean Combs', 'music.artist.genre', 'Pop music'), ('Hold Tight', 'music.composition.recordings', 'Hold Tight'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Kelly Clarkson', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'music.album.artist', 'Far East Movement'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('World music', 'broadcast.genre.content', 'BeirutNights.com Radio'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Nicki Minaj', 'people.person.profession', 'Singer-songwriter'), ('HitzRadio.com', 'broadcast.content.broadcast', 'HitzRadio.com - 128kbps Stream'), ('Max Martin', 'music.artist.genre', 'Europop'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.artist', 'Justin Bieber'), ('Jordan Francis', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Chris Jasper', 'music.composer.compositions', 'Bad Day'), ('Tricky Stewart', 'people.person.profession', 'Musician'), ('Somebody to Love', 'music.composition.composer', 'Heather Bright'), ('English Language', 'language.human_language.countries_spoken_in', 'Canada'), ('Music Producer', 'common.topic.notable_types', 'Profession'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Chris Brown', 'music.artist.genre', 'Dance music'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO Todays RNB'), ('181-beat', 'broadcast.content.genre', 'Contemporary R&B'), ('Brandy Norwood', 'people.person.profession', 'Actor'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Somebody to Love (J Stax remix)', 'music.recording.canonical_version', 'Somebody To Love'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Children'), ('m.0njvs9s', 'award.award_honor.award_winner', 'Justin Bieber'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_svs8m'), ('m.0gctps1', 'tv.tv_guest_role.episodes_appeared_in', 'Miranda Hart, Russell Howard, Justin Bieber, David Haye and Florence and the Machine'), ('Pray', 'music.recording.artist', 'Justin Bieber'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhyh_'), ('Jennifer Lopez', 'music.artist.label', 'Island Records'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw59_'), ('Kanye West', 'music.artist.genre', 'Pop music'), ('Kanye West', 'music.artist.genre', 'Rhythm and blues'), ('Somebody to Love (remix)', 'common.topic.notable_types', 'Musical Album'), ('Bad Day', 'music.album.artist', 'Justin Bieber'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.recording.song', 'Baby'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'M.I.A.'), ('Gwen Stefani', 'people.person.profession', 'Artist'), ('181-beat', 'broadcast.content.artist', 'Usher'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Official website'), ('m.012bm2cn', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Soulja Boy', 'people.person.gender', 'Male'), ('m.0yqflhy', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Artist'), ('m.07053l5', 'common.webpage.topic', 'Justin Bieber'), ('m.0y7y53r', 'award.award_honor.ceremony', 'Kerrang! Awards 2012'), ('Kelly Clarkson', 'music.artist.genre', 'Contemporary R&B'), ('Dapo Torimiro', 'music.composer.compositions', 'Bigger'), ('m.0gbm3bl', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('The Island Def Jam Music Group', 'organization.organization.previous_names', 'm.05sp41b'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Classic hits'), ('Rita Ora', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'), ('Lolly', 'music.album.primary_release', 'Lolly'), ('Marvin Isley', 'people.person.nationality', 'United States of America'), ('The Black Eyed Peas', 'music.artist.genre', 'Dance music'), ('m.0z8755b', 'award.award_honor.ceremony', 'NME Awards 2012'), ('Miley Cyrus', 'people.person.profession', 'Dancer'), ('Justin Bieber: Just Getting Started', 'book.written_work.original_language', 'English Language'), ('First Dance', 'music.composition.composer', 'Usher'), ('As Long As You Love Me (Audiobot instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('Zac Brown Band', 'common.topic.notable_types', 'Musical Artist'), ('Donna Summer', 'music.artist.origin', 'United States of America'), ('Jennifer Lopez', 'music.artist.genre', 'Pop music'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('m.0cq990t', 'common.webpage.topic', 'Justin Bieber'), ('Bad 25', 'film.film.language', 'English Language'), ('Shaggy', 'freebase.valuenotation.has_value', 'Parents'), ('Eminem', 'common.topic.notable_types', 'Musical Artist'), ('Hit-Boy', 'common.topic.notable_types', 'Musical Artist'), ('Beautiful', 'common.topic.notable_types', 'Musical Recording'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.episode', 'Series 1 Episode 18'), ('Beyoncé Knowles', 'music.artist.genre', 'Electronic dance music'), ('Baby', 'music.album.compositions', 'Baby'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Rita Ora', 'music.artist.genre', 'Rhythm and blues'), ('Hit-Boy', 'people.person.profession', 'Record producer'), ('Change Me', 'common.topic.notable_types', 'Musical Album'), ('Avril Lavigne', 'music.artist.genre', 'Pop music'), ('Boyfriend', 'common.topic.notable_for', 'g.1259shy1p'), ('Daniel Bedingfield', 'music.artist.label', 'The Island Def Jam Music Group'), ('Live My Life', 'common.topic.notable_types', 'Musical Album'), ('Estelle', 'common.topic.notable_types', 'Musical Artist'), ('Music', 'common.topic.subjects', 'Artfarm'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ashley Tisdale'), ('m.0njhvsq', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'Randy Phillips'), ('Shaun Cassidy', 'people.person.profession', 'Actor'), ('Jason McCann', 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr28j'), ('Contemporary R&B', 'broadcast.genre.content', 'JellyRadio.com'), ('radioIO RNB Mix', 'broadcast.content.genre', \"00's\"), ('Redfoo', 'people.person.profession', 'Dancer'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: V101', 'broadcast.content.artist', 'Santana'), ('Yves Bole', 'people.person.profession', 'Model'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Jason Mraz'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0zbf_4g'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjhb'), ('Rihanna', 'broadcast.artist.content', 'radioIO Todays POP'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rodney Jerkins', 'people.person.gender', 'Male'), ('Montell Jordan', 'people.person.nationality', 'United States of America'), ('Duffy', 'music.artist.genre', 'Rhythm and blues'), ('181-beat', 'broadcast.content.artist', 'Twista'), ('Camagüey', 'location.location.people_born_here', 'Yves Bole'), ('PowerHitz', 'broadcast.content.artist', 'Chris Brown'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kid Cudi'), ('Kings of Leon', 'broadcast.artist.content', 'Hot Wired Radio'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rob Thomas'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber', 'base.popstra.celebrity.dated', 'm.0gxnp02'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Buckcherry'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Country of nationality'), ('2011 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0115qhzk'), ('Recovery', 'music.album.artist', 'Justin Bieber'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Author', 'freebase.type_hints.included_types', 'Topic'), ('Rudolph Isley', 'freebase.valuenotation.has_value', 'Parents'), ('My Worlds: The Collection', 'music.album.artist', 'Justin Bieber'), ('Avery', 'music.artist.genre', 'Pop music'), ('Nelly Furtado', 'music.artist.genre', 'World music'), ('m.0yrkc0l', 'award.award_honor.award_winner', 'Justin Bieber'), ('Sheryl Crow', 'broadcast.artist.content', 'HitzRadio.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Fergie'), ('Keyshia Cole', 'people.person.profession', 'Singer'), ('Justin Bieber', 'music.artist.album', 'Somebody to Love (remix)'), ('August Rigo', 'music.artist.genre', 'Pop music'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('radioIO Classic RNB', 'common.topic.notable_types', 'Broadcast Content'), ('The Black Eyed Peas', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkw8v'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.artist', 'Justin Bieber'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Urban contemporary'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten remix)'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z2dr9y'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('m.0_x3kvk', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Avril Lavigne', 'broadcast.artist.content', 'radioIO Todays POP'), ('Will Smith', 'music.artist.genre', 'Urban contemporary'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('CL', 'music.artist.genre', 'Contemporary R&B'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Athan Grace'), ('Demi Lovato', 'music.artist.genre', 'Rhythm and blues'), ('Billboard Music Award for Top Streaming Song (Video)', 'award.award_category.winners', 'm.0njhyh_'), ('Somebody to Love', 'common.topic.notable_for', 'g.12h2wlms9'), ('Baby', 'music.album.genre', 'Electronic music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvvj_'), ('Usher', 'people.person.profession', 'Film Producer'), ('All Around the World', 'music.single.versions', 'All Around the World (acoustic version)'), ('Chris Brown', 'people.person.languages', 'English Language'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Classic hits'), ('m.0z8s562', 'award.award_honor.award_winner', 'Justin Bieber'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Contemporary R&B'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Günther & The Sunshine Girls'), ('My World', 'music.album.releases', 'My World'), ('m.0rqp4h0', 'music.track_contribution.contributor', 'Justin Bieber'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Height'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsk'), ('Raekwon', 'people.person.gender', 'Male'), ('HitzRadio.com', 'broadcast.content.artist', 'Dr. Dre'), ('m.0njvtth', 'award.award_honor.award_winner', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Ludacris'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92p'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Akon'), ('Red Hot Chili Peppers', 'common.topic.notable_types', 'Musical Artist'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Island Def Jam Music Group', 'common.topic.webpage', 'm.064xzb1'), ('Savan Kotecha', 'music.artist.genre', 'Teen pop'), ('Believe', 'music.album.release_type', 'Album'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.tracks', 'Wont Stop (feat. Justin Bieber)'), ('Spouse (or domestic partner)', 'rdf-schema#range', 'Marriage'), ('PowerHitz', 'broadcast.content.artist', 'Juvenile'), ('Khalil', 'music.artist.genre', 'Contemporary R&B'), ('radioIO Todays RNB', 'broadcast.content.location', 'Tampa'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('Justin Bieber', 'music.artist.album', 'My Worlds Acoustic'), ('Cris Cab', 'people.person.gender', 'Male'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bq275'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Phoenix'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('Scooter Braun', 'people.person.nationality', 'United States of America'), ('Runaway Love (remix)', 'common.topic.notable_for', 'g.1yg9g44l8'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Johnny Crawford', 'people.person.profession', 'Singer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Rap music'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (acoustic version)'), ('Bryan Adams', 'people.person.profession', 'Actor'), ('Nasri', 'people.person.place_of_birth', 'Canada'), ('Daniel Bedingfield', 'music.artist.genre', 'Electronic music'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Chingy'), ('Heartbreaker', 'music.album.compositions', 'Heartbreaker'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0pcnqnb'), ('Donna Summer', 'music.artist.genre', 'Rhythm and blues'), ('Ricky Nelson', 'people.person.nationality', 'United States of America'), ('1Club.FM: Channel One', 'broadcast.content.producer', '1Club.FM'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Shaggy'), ('Justin Bieber', 'broadcast.artist.content', '.977 The Hits Channel'), ('Never Say Never: The Remixes', 'music.album.album_content_type', 'Remix album'), ('HitzRadio.com', 'broadcast.content.artist', 'Mary J. Blige'), ('Miley Cyrus', 'broadcast.artist.content', 'radioIO Todays POP'), ('#thatPOWER', 'music.single.versions', '#Thatpower'), ('Usher', 'broadcast.artist.content', 'FLOW 103'), ('Beauty and a Beast', 'common.topic.notable_types', 'Musical Recording'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mary J. Blige'), ('1Club.FM: Power', 'broadcast.content.artist', 'Ja Rule'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life (Party Rock remix)'), ('American Music Awards of 2010', 'award.award_ceremony.nominees', 'm.0zbg2kw'), ('Shaffer Smith', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Lupe Fiasco', 'people.person.profession', 'Singer'), ('Gwen Stefani', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Synthpop', 'music.genre.parent_genre', 'Rock music'), ('Sheryl Crow', 'people.person.nationality', 'United States of America'), ('Demi Lovato', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Alicia Keys', 'broadcast.artist.content', 'FLOW 103'), ('Gwen Stefani', 'people.person.profession', 'Actor'), ('Ontario', 'location.administrative_division.first_level_division_of', 'Canada'), ('The Isley Brothers', 'music.artist.label', 'Island Records'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Next'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Year'), ('1.FM Top 40', 'broadcast.content.artist', 'Puddle of Mudd'), ('All That Matters (video)', 'music.recording.song', 'All That Matters'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Tommy Sands', 'people.person.place_of_birth', 'Chicago'), ('Justin bieber', 'common.image.appears_in_topic_gallery', 'Justin Bieber'), ('m.0yqfny6', 'award.award_honor.award', 'MTV Europe Music Award for Best Canadian Act'), ('Usher', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Redfoo', 'common.topic.notable_types', 'Musical Artist'), ('School Boy Records', 'organization.organization.founders', 'Scooter Braun'), ('Disney Parks Christmas Day Parade', 'film.film.genre', 'Music'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv97'), ('United States Dollar', 'finance.currency.countries_used', 'United States of America'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9hp2'), ('Alicia Keys', 'people.person.profession', 'Musician'), ('Whitney Houston', 'music.artist.genre', 'Rhythm and blues'), ('Victoria Justice', 'music.artist.genre', 'Dance-pop'), ('Contemporary R&B', 'common.topic.article', 'm.025sc53'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0yrk0mt', 'award.award_honor.award_winner', 'Justin Bieber'), ('GotRadio - RnB Classics', 'broadcast.content.genre', 'Contemporary R&B'), ('50 Cent', 'broadcast.artist.content', 'FLOW 103'), ('All Around the World (acoustic version)', 'music.recording.canonical_version', 'All Around the World'), ('Jazmyn Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Katy Perry', 'people.person.gender', 'Female'), ('Dr. Dre', 'people.person.profession', 'Actor'), ('Trick Daddy', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0ywtfj6', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Official website', 'type.property.schema', 'Topic'), ('Beyoncé Knowles', 'broadcast.artist.content', 'HitzRadio.com'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Shiny Toy Guns'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Believe', 'music.album.supporting_tours', 'Believe Tour'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Nasri', 'music.composer.compositions', 'Pray'), ('Trick Daddy', 'people.person.place_of_birth', 'Miami'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvkl'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Everlast'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Jordin Sparks', 'broadcast.artist.content', '.977 The Hits Channel'), ('1.FM Top 40', 'broadcast.content.artist', 'U2'), ('m.0z0tmyv', 'award.award_honor.award', 'Shorty Award for Music'), ('Jennifer Lopez', 'people.person.profession', 'Record producer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Jay-Z'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Cascada'), ('Athan Grace', 'people.person.gender', 'Male'), ('m.0zg2w8r', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Cassie Ventura'), ('Reed Smoot', 'film.cinematographer.film', 'Justin Bieber: Never Say Never'), ('m.0yrjynf', 'award.award_honor.award_winner', 'Justin Bieber'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6_ww'), ('Little Bird', 'music.recording.artist', 'Far East Movement'), ('Scooter Braun', 'organization.organization_founder.organizations_founded', 'School Boy Records'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Calling'), ('1Club.FM: V101', 'broadcast.content.producer', '1Club.FM'), ('New Kids on the Block', 'broadcast.artist.content', '1Club.FM: Channel One'), ('1Club.FM: V101', 'broadcast.content.artist', 'Leona Lewis'), ('Geri Halliwell', 'people.person.gender', 'Female'), ('m.0yrk0mt', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Christina Aguilera', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.0101ftww', 'film.film_cut.film', \"Justin Bieber's Believe\"), ('Raekwon', 'music.featured_artist.albums', 'Runaway Love (remix)'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Parents'), ('All That Matters (video)', 'music.recording.canonical_version', 'All That Matters'), ('Recovery', 'music.album.releases', 'Recovery'), ('Jennifer Lopez', 'broadcast.artist.content', 'Sunshine Radio'), ('Never Say Never: The Remixes', 'music.album.genre', 'Teen pop'), ('Montell Jordan', 'music.artist.genre', 'Contemporary R&B'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njgyk4'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Terius Nash', 'common.topic.notable_types', 'Musical Artist'), ('Urban contemporary', 'broadcast.genre.content', 'FLOW 103'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_3hn'), ('HitzRadio.com', 'broadcast.content.artist', 'Sheryl Crow'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Parents'), ('Aaliyah', 'broadcast.artist.content', 'radioIO RNB Mix'), ('m.0nh04xx', 'film.personal_film_appearance.film', 'Justin Bieber: Rise to Fame'), ('P!nk', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('m.011m26pv', 'music.group_membership.member', 'Justin Bieber'), ('Gwen Stefani', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Gender'), ('UR Fave: New Artist', 'award.award_category.winners', 'm.0njwb81'), ('m.0pcr2qn', 'tv.tv_guest_role.episodes_appeared_in', 'Episode #2.10'), ('Jason Mraz', 'people.person.languages', 'English Language'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Parents'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Hip hop music'), ('Lady Gaga', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jennifer Lopez', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Outkast'), ('Mariah Carey', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('HitzRadio.com', 'common.topic.article', 'm.03hs0fv'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Don Henley', 'people.person.profession', 'Musician'), ('Barry Weiss', 'music.artist.genre', 'Pop music'), ('Contemporary R&B', 'broadcast.genre.content', 'C9 Radio'), ('Sheryl Crow', 'broadcast.artist.content', 'Sunshine Radio'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ciara'), ('R. Kelly', 'music.artist.genre', 'Hip hop music'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Pop music'), ('Timbaland', 'people.person.gender', 'Male'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gts1'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Geri Halliwell', 'music.artist.genre', 'Synthpop'), ('Athan Grace', 'people.person.gender', 'Male'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jordin Sparks'), ('SoulfulClassics.com', 'common.topic.notable_types', 'Broadcast Content'), ('Ronski Speed', 'people.person.gender', 'Male'), ('Shaggy', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mann'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('m.0y4tdml', 'award.award_honor.ceremony', '2013 Radio Disney Music Awards'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Johnny Crawford', 'people.person.profession', 'Actor'), ('Stuart Ford', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ciara', 'broadcast.artist.content', 'PowerHitz'), ('1.FM Top 40', 'broadcast.content.artist', 'Rascal Flatts'), ('World music', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.featured_artists', 'Big Sean'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.featured_artists', 'Big Sean'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Children'), ('Yves Bole', 'base.musicpf.artist.album', 'Raindrops'), ('Miami Beach', 'location.location.time_zones', 'Eastern Time Zone'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012bq275'), ('Teyana', 'people.person.profession', 'Actor'), ('Wait For a Minute', 'music.album.artist', 'Tyga'), ('Rudolph Valentino', 'people.person.gender', 'Male'), ('m.0njwb81', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Usher', 'music.artist.genre', 'Pop music'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('My Worlds', 'music.album.primary_release', 'My Worlds'), ('Justin Bieber', 'music.artist.album', 'Confident'), ('Singer-songwriter', 'people.profession.part_of_professional_field', 'Singing'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('The Pussycat Dolls', 'broadcast.artist.content', 'FLOW 103'), ('As Long as You Love Me', 'music.album.release_type', 'Single'), ('Believe', 'common.topic.notable_types', 'Musical Album'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Ludacris'), ('Pray', 'music.composition.recordings', 'Pray'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Musical Album', 'type.type.properties', 'Initial release date'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('Gavin DeGraw', 'music.artist.genre', 'Acoustic music'), ('Shaggy', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Big Pun'), ('Teyana', 'common.topic.notable_types', 'Musical Artist'), ('Ronski Speed', 'common.topic.notable_types', 'Musical Artist'), ('Beyoncé Knowles', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Nelly', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0vpgdpb', 'music.recording_contribution.album', '#thatPOWER'), ('Duffy', 'people.person.profession', 'Actor'), ('#thatPower', 'music.composition.recordings', '#thatPower'), ('Timbaland', 'people.person.profession', 'Singer'), ('Roller Coaster', 'music.album.releases', 'Roller Coaster'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Artist'), ('m.0pcr28j', 'tv.tv_guest_role.character', 'Jason McCann'), ('m.0101fvcp', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('PYD', 'award.award_nominated_work.award_nominations', 'm.0_x3kvk'), ('Mistletoe', 'music.album.compositions', 'Mistletoe'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'JoJo'), ('Enrique Iglesias', 'freebase.valuenotation.has_no_value', 'Children'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('Jon M. Chu', 'people.person.profession', 'Actor'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw257'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9hvh'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d3'), ('m.0njhxzc', 'award.award_honor.award', 'Billboard Music Award for Top Pop Album'), ('Right Here', 'music.single.versions', 'Right Here'), ('Height', 'rdf-schema#domain', 'Person'), ('Cory Gunz', 'common.topic.notable_types', 'Musical Artist'), ('Frank Sinatra', 'base.icons.icon.icon_genre', 'Teen idol'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Dance music'), ('Cris Cab', 'music.artist.genre', 'Pop music'), ('Singer', 'people.profession.specializations', 'Child singer'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('m.0101ft0d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Hot 108 Jamz', 'broadcast.content.artist', 'Rihanna'), ('David Cassidy', 'people.person.profession', 'Singer'), ('First Dance', 'music.recording.song', 'First Dance'), ('m.0101fvp7', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Baby', 'music.album.release_type', 'Single'), ('Live My Life', 'music.composition.recordings', 'Little Bird'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Singer', 'people.profession.specializations', 'Sufi singer'), ('As Long as You Love Me', 'music.composition.composer', 'Rodney Jerkins'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Ciara'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Stephen Melton', 'common.topic.subjects', 'Rock music'), (\"O'Kelly Isley, Jr.\", 'people.person.gender', 'Male'), ('Geri Halliwell', 'people.person.profession', 'Singer-songwriter'), ('m.0101fvk9', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0t4s_bn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Reed Smoot', 'people.person.place_of_birth', 'United States of America'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Nelly Furtado'), ('C1', 'music.artist.label', 'The Island Def Jam Music Group'), ('Usher', 'music.lyricist.lyrics_written', 'First Dance'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gz0fr6'), ('Bad Day', 'music.composition.composer', 'Ernie Isley'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fszs'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0tjyljn', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Tricky Stewart', 'music.producer.releases_produced', 'My World'), ('Ludacris', 'people.person.profession', 'Film Producer'), ('Heartbreaker', 'music.composition.composer', 'Justin Bieber'), ('Christina Milian', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft1r'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.winners', 'm.0wjgqck'), ('Bryan Adams', 'broadcast.artist.content', '1.FM Top 40'), ('School Boy Records', 'common.topic.notable_types', 'Record label'), ('Will i Am', 'people.person.gender', 'Male'), ('m.0yrhmll', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0pc670l'), ('1Club.FM: V101', 'broadcast.content.artist', 'Chris Brown'), ('Madonna', 'music.artist.genre', 'Dance-pop'), (\"O'Kelly Isley, Jr.\", 'people.person.nationality', 'United States of America'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Clay Aiken', 'people.person.profession', 'Singer'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Jazmyn Bieber', 'people.person.gender', 'Female'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cute Is What We Aim For'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Right Here'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Frank Ocean'), ('#Thatpower', 'music.recording.featured_artists', 'Justin Bieber'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.04kjw8n'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Alice Deejay'), ('J. Holiday', 'music.artist.genre', 'Pop music'), ('Change Me', 'music.recording.artist', 'Justin Bieber'), ('RBMG Records', 'common.topic.notable_for', 'g.12579f_0q'), ('m.0gz0fr6', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Rhythm and blues'), ('Reggae', 'broadcast.genre.content', 'JellyRadio.com'), ('Change Me', 'freebase.valuenotation.is_reviewed', 'Composer'), ('1.FM Top 40', 'broadcast.content.artist', 'Christina Aguilera'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Chris Brown'), (\"2012 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkw_d'), ('Fall Out Boy', 'music.artist.label', 'Island Records'), ('m.0_x3kvk', 'award.award_nomination.nominated_for', 'PYD'), ('Ludacris', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0_cz5l3', 'celebrities.legal_entanglement.location', 'Miami Beach'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Marvin Isley', 'people.deceased_person.place_of_death', 'Chicago'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Judy Garland', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fvps', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Zac Efron', 'people.person.profession', 'Singer'), ('Justin Bieber', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Pras'), ('FLOW 103', 'broadcast.content.genre', 'Hip hop music'), ('m.0gctytd', 'tv.tv_guest_role.episodes_appeared_in', 'January 15, 2010'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('United States of America', 'common.topic.notable_types', 'Country'), ('m.0pcnqnb', 'film.personal_film_appearance.film', \"Dick Clark's Primetime New Year's Rockin' Eve 2013\"), ('Stratford Northwestern Secondary School', 'education.educational_institution.students_graduates', 'm.0yrlqp1'), ('Jennifer Lopez', 'people.person.gender', 'Female'), ('Nick Jonas', 'people.person.nationality', 'United States of America'), ('Beautiful and the Beat', 'music.recording.song', 'Beauty And A Beat'), ('Nicki Minaj', 'music.artist.genre', 'Pop music'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Yves Bole', 'influence.influence_node.influenced', 'Rihanna'), ('m.0njw1tn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kelly Clarkson'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfpbq5'), ('m.0yr9c1k', 'award.award_honor.honored_for', 'My World'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.010lkp2z', 'award.award_honor.ceremony', '2014 Billboard Music Awards'), ('Estelle', 'music.artist.genre', 'Electronic music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Estelle', 'music.artist.genre', 'Contemporary R&B'), ('Believe Acoustic', 'common.topic.notable_types', 'Musical Album'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Burnham'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('1Club.FM: Channel One', 'common.topic.image', '1clubfm.jpg'), ('All Around The World', 'award.award_nominated_work.award_nominations', 'm.0z85n_1'), ('Little Bird', 'music.recording.song', 'Live My Life'), ('Singer-songwriter', 'common.topic.subject_of', 'Stephen Melton'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Gender'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ajda Pekkan'), ('Rihanna', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.artist', 'Justin Bieber'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Donna Summer', 'music.artist.genre', 'Dance-pop'), ('Donna Summer', 'common.topic.notable_types', 'Musical Artist'), ('m.012bm4v7', 'celebrities.friendship.friend', 'Yves Bole'), ('Under the Mistletoe', 'music.album.primary_release', 'Under the Mistletoe'), ('Don Henley', 'music.artist.genre', 'Pop music'), ('LeAnn Rimes', 'people.person.nationality', 'United States of America'), ('m.0ywvh8k', 'award.award_honor.ceremony', '5th Annual Shorty Awards'), ('Tyga', 'music.artist.genre', 'Hip hop music'), ('m.0yr9c1k', 'award.award_honor.award_winner', 'Justin Bieber'), ('Wait for a Minute', 'award.award_nominated_work.award_nominations', 'm.0_xlw5f'), ('Recovery', 'music.album.release_type', 'Single'), ('Lolly', 'music.album.artist', 'Maejor'), ('Rob Thomas', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)'), ('Justin Bieber: Rise to Fame', 'film.film.language', 'English Language'), ('Musical Album', 'type.type.properties', 'Release type'), ('Terence Dudley', 'common.topic.notable_types', 'Record Producer'), ('Where Are Ü Now', 'music.recording.featured_artists', 'Justin Bieber'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Flyleaf'), ('m.0njw444', 'award.award_honor.award_winner', 'Justin Bieber'), ('Clay Aiken', 'people.person.profession', 'Singer-songwriter'), (\"Turn to You (Mother's Day Dedication)\", 'music.recording.artist', 'Justin Bieber'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Maurice Steenbergen'), ('JellyRadio.com', 'broadcast.content.artist', 'Tupac Shakur'), ('m.0z1scxk', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Shaggy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Mariah Carey', 'broadcast.artist.content', '1.FM Top 40'), ('Hip hop music', 'broadcast.genre.content', '181-thebox'), ('R. Kelly', 'common.topic.notable_types', 'Musical Artist'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njgyk4'), ('Sean Combs', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Deborah Lurie', 'people.person.nationality', 'United States of America'), ('Chris Brown', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Red Jumpsuit Apparatus'), ('Victoria Justice', 'people.person.profession', 'Dancer'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Rock music', 'common.topic.notable_types', 'Musical genre'), ('Where Are Ü Now', 'music.recording.tracks', 'g.11bv1nyzls'), ('Hit-Boy', 'music.composer.compositions', 'Right Here'), ('All Bad', 'common.topic.notable_for', 'g.1yn5bwn26'), ('Verse Simmonds', 'people.person.profession', 'Singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Justin Timberlake'), ('Cris Cab', 'people.person.profession', 'Musician'), ('m.09y8bsk', 'common.webpage.topic', 'Teen idol'), ('PowerHitz', 'broadcast.content.artist', 'Akon'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sisqó'), ('Soulja Boy', 'music.artist.genre', 'Dance music'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Blackstreet', 'music.artist.genre', 'Contemporary R&B'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0101ft5f', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('All Around The World', 'music.composition.composer', 'Nasri'), ('Will Smith', 'people.person.gender', 'Male'), ('Leif Garrett', 'people.person.profession', 'Actor'), ('m.0gbm3d_', 'film.personal_film_appearance.person', 'Jaden Smith'), ('Enrique Iglesias', 'people.person.profession', 'Actor'), ('Usher', 'people.person.profession', 'Dancer'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98_q'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Colbie Caillat', 'people.person.profession', 'Singer-songwriter'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.featured_artists', 'Big Sean'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Record producer', 'common.topic.webpage', 'm.09wjs_l'), ('m.09wmk1t', 'common.webpage.topic', 'Teen pop'), ('m.0gxnp72', 'base.popstra.support.supporter', 'Justin Bieber'), ('Coldplay', 'broadcast.artist.content', '.977 The Hits Channel'), ('As Long As You Love Me (Audien dubstep mix)', 'common.topic.notable_types', 'Musical Recording'), ('Baby', 'music.composition.recordings', 'Baby'), ('#thatPOWER', 'common.topic.notable_for', 'g.12mq3wvxs'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0v90p2b'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Album'), ('Jason Mraz', 'people.person.profession', 'Musician'), ('Sean Combs', 'people.person.profession', 'Record producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Lady Sovereign'), ('Yves Bole', 'common.topic.notable_types', 'Musical Artist'), ('Kelly Clarkson', 'music.artist.genre', 'Rock music'), ('Die in Your Arms', 'common.topic.notable_for', 'g.126skfrp3'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fszb'), ('Ronald Isley', 'people.person.profession', 'Singer-songwriter'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Rock music'), ('m.0101fvkl', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Contemporary R&B', 'broadcast.genre.content', 'DeeGay'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'DMX'), ('Record producer', 'base.schemastaging.context_name.pronunciation', 'g.125_lq82d'), ('Rob Thomas', 'broadcast.artist.content', 'radioIO Todays POP'), ('#Thatpower', 'music.recording.song', '#thatPower'), ('HitzRadio.com', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Vanessa Hudgens', 'music.artist.genre', 'Contemporary R&B'), ('m.0_grqb8', 'celebrities.friendship.friend', 'Taylor Swift'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'Recovery'), ('Stephen Melton', 'common.topic.subjects', 'Musician'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'award.award_winning_work.awards_won', 'm.0sgk_cw'), ('Jay-Z', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Fall Out Boy'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vz9'), ('Juelz Santana', 'people.person.nationality', 'United States of America'), ('2012 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('Colbie Caillat', 'people.person.profession', 'Artist'), ('m.0zbg2kw', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0gbm3c3', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Justin Bieber'), ('Beauty and a Beast', 'music.recording.song', 'Beauty And A Beat'), ('Parents', 'owl#inverseOf', 'Children'), ('PYD', 'music.composition.recordings', 'PYD'), ('Chicago', 'base.biblioness.bibs_location.country', 'United States of America'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.appearance_type', 'Him/Herself'), ('Hit-Boy', 'people.person.nationality', 'United States of America'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'DJ Cyglas vs. DJ Pure'), ('JoJo', 'broadcast.artist.content', 'radioIO Todays POP'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('radioIO Classic RNB', 'broadcast.content.location', 'Tampa'), ('Justin Timberlake', 'people.person.nationality', 'United States of America'), ('Roller Coaster', 'music.album.primary_release', 'Roller Coaster'), ('Believe', 'common.topic.notable_for', 'g.1259h0wh_'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Pop music'), ('All Around the World', 'common.topic.notable_types', 'Musical Album'), ('m.0gfpb_k', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('m.0sxhq3d', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien dubstep mix)'), ('m.0tk76mf', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('#Thatpower', 'common.topic.notable_types', 'Musical Recording'), ('Never Say Never', 'music.recording.artist', 'Justin Bieber'), ('Lupe Fiasco', 'people.person.profession', 'Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Top 40'), ('Justin Bieber: Just Getting Started', 'book.book.editions', 'Justin Bieber: Just Getting Started'), ('My Worlds: The Collection', 'common.topic.notable_for', 'g.125cvdbq7'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0r90dwg', 'common.webpage.resource', 'Justin Bieber Concerts'), ('L.A. Reid', 'music.artist.label', 'The Island Def Jam Music Group'), ('David Cassidy', 'people.person.nationality', 'United States of America'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsr'), ('Kings of Leon', 'music.artist.genre', 'Rock music'), ('PowerHitz', 'broadcast.content.artist', 'Shaffer Smith'), ('Johntá Austin', 'people.person.profession', 'Singer-songwriter'), ('Boyfriend', 'common.topic.notable_for', 'g.12590k9tt'), ('1Club.FM: Power', 'broadcast.content.artist', 'Shaggy'), ('Singer', 'people.profession.specializations', 'Ghost singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Radiohead'), ('Adam Messinger', 'music.artist.genre', 'Rock music'), ('Lolly', 'common.topic.notable_for', 'g.1yg94clyx'), ('m.0nh04xx', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beauty and a Beat (Remixes)', 'common.topic.notable_types', 'Musical Album'), ('Recovery', 'music.composition.recordings', 'Recovery'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaun Cassidy', 'people.person.profession', 'Singer'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Profession'), ('HitzRadio.com', 'broadcast.content.artist', 'Shaffer Smith'), ('Yves Bole', 'people.person.place_of_birth', 'Camagüey'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Linkin Park', 'broadcast.artist.content', 'radioIO Todays POP'), ('Mistletoe', 'common.topic.notable_for', 'g.12h2xc_71'), ('Yves Bole', 'influence.influence_node.influenced_by', 'Justin Bieber'), ('Hot 108 Jamz', 'common.topic.notable_types', 'Broadcast Content'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Adrienne Bailon', 'people.person.gender', 'Female'), ('HitzRadio.com', 'broadcast.content.artist', 'Terius Nash'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Twista'), ('Miami', 'location.location.time_zones', 'Eastern Time Zone'), ('m.0z83x9l', 'award.award_honor.ceremony', 'NME Awards 2012'), ('Jordan Francis', 'music.artist.genre', 'Dance-pop'), ('Kerrang! Villain of the Year Award', 'award.award_category.winners', 'm.0y7y53r'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Official website'), ('English Language', 'language.human_language.main_country', 'United States of America'), ('Chris Jasper', 'music.artist.genre', 'Rhythm and blues'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw444'), ('m.0zbf_4g', 'award.award_honor.award_winner', 'Justin Bieber'), ('Kelly Clarkson', 'music.artist.genre', 'Pop music'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_72tb'), ('m.0z84kwx', 'award.award_nomination.award', 'Teen Choice Award for Choice Single: Male Artist'), ('m.0vmyv4w', 'music.recording_contribution.contributor', 'Ludacris'), ('Sheryl Crow', 'people.person.gender', 'Female'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The All-American Rejects'), ('The Roots', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0yrktlv', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Runaway Love', 'music.single.versions', 'Runaway Love (remix)'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO Todays POP'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0k05qcr'), ('Whitney Houston', 'people.person.profession', 'Artist'), ('Fergie', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3bs'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Children'), ('Journals', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Ellen DeGeneres', 'people.person.profession', 'Film Producer'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Jordin Sparks'), ('m.0sgkyfg', 'award.award_honor.ceremony', \"2011 Kids' Choice Awards\"), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Pussycat Dolls'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Milk & Sugar'), ('m.0d33gyj', 'celebrities.romantic_relationship.celebrity', 'Caitlin Beadles'), ('m.0w5l5h1', 'freebase.valuenotation.has_value', 'Start Date'), ('Usher', 'music.featured_artist.albums', 'Somebody to Love (remix)'), ('Nicki Minaj', 'freebase.valuenotation.has_no_value', 'Children'), ('m.0wjgrzc', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('m.0d33gyj', 'celebrities.romantic_relationship.celebrity', 'Justin Bieber'), ('Place of birth', 'owl#inverseOf', 'People born here'), ('Jessica Simpson', 'music.artist.genre', 'Synthpop'), ('Kid Cudi', 'broadcast.artist.content', '1Club.FM: Power'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_729v'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jeremy Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0101fs_z'), ('Justin bieber', 'common.image.size', 'm.01x59ss'), ('Terence Dudley', 'music.artist.genre', 'Pop music'), ('m.0vb6hhj', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Ice Cube', 'broadcast.artist.content', 'JellyRadio.com'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cx'), ('Ja Rule', 'broadcast.artist.content', '1Club.FM: Power'), ('Janet Jackson', 'music.artist.genre', 'Rock music'), ('m.0yrkgd6', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Beauty and a Beat', 'common.topic.notable_for', 'g.126stnfcp'), ('Carrie Underwood', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Rodney Jerkins', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft2j'), ('radioIO Todays POP', 'common.topic.image', 'radioio2.gif'), ('Adam Messinger', 'people.person.gender', 'Male'), ('m.0_vmmj6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('RedOne', 'people.person.profession', 'Record producer'), ('Ashley Tisdale', 'people.person.nationality', 'United States of America'), ('Janet Jackson', 'music.artist.genre', 'Dance-pop'), ('Kylie Minogue', 'people.person.profession', 'Artist'), ('Never Say Never', 'music.recording.song', 'Never Say Never'), ('Ashlee Simpson', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'common.topic.webpage', 'm.0cq990t'), ('m.0z8755b', 'award.award_honor.honored_for', 'Under the Mistletoe'), ('Shaffer Smith', 'broadcast.artist.content', 'FLOW 103'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0n4rmg7'), ('Young Artists for Haiti', 'music.musical_group.member', 'm.011m26pv'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sara Bareilles'), ('Whitney Houston', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sisqó'), ('Hikaru Utada', 'people.person.languages', 'English Language'), ('Ronald Isley', 'people.person.gender', 'Male'), ('m.0z898w6', 'award.award_honor.award_winner', 'Justin Bieber'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO Todays POP'), ('Ricky Nelson', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Tom Cloud'), ('Teen Choice Award for Choice Red Carpet Fashion Icon Male', 'award.award_category.winners', 'm.0z87d3n'), ('Brit Award for International Breakthrough Act', 'award.award_category.winners', 'm.0y803nt'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Demi Lovato'), ('Book', 'freebase.type_profile.strict_included_types', 'Written Work'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Capone'), ('Musician', 'people.profession.specializations', 'Singer-songwriter'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Thought Of You', 'common.topic.notable_for', 'g.125525fwv'), ('Right Here', 'music.recording.artist', 'Drake'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0n1ykxp'), ('Coldplay', 'music.artist.genre', 'Indie rock'), ('181-beat', 'broadcast.content.genre', 'Hip hop music'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Dance music'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Estelle', 'music.artist.genre', 'Pop music'), ('m.0101ftq9', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Pras', 'common.topic.notable_types', 'Musical Artist'), ('1Club.FM: 80s (Pop)', 'common.topic.image', '1clubfm.jpg'), ('Justin Bieber', 'celebrities.celebrity.legal_entanglements', 'm.0_cyzs_'), ('Fabian', 'people.person.profession', 'Actor'), ('m.0101fv8j', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('Die in Your Arms', 'music.recording.releases', 'Believe'), ('My World', 'award.award_winning_work.awards_won', 'm.0yr9c1k'), ('Alanis Morissette', 'people.person.profession', 'Film Producer'), ('Canadian', 'common.topic.notable_for', 'g.1255pjt30'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Alicia Keys'), ('m.0j_tkcq', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Never Say Never', 'common.topic.notable_types', 'Musical Album'), ('m.0gc_9w6', 'common.webpage.resource', 'Justin Bieber Videos'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z7n7h'), ('Anastacia', 'common.topic.notable_types', 'Musical Artist'), ('m.0gxnny8', 'people.place_lived.person', 'Justin Bieber'), ('P!nk', 'music.artist.genre', 'Hip hop music'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Pray'), ('P!nk', 'broadcast.artist.content', 'Sunshine Radio'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m5x'), ('Hold Tight', 'music.composition.composer', 'James Giannos'), ('Britney Spears', 'music.artist.genre', 'Electronic music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z898w6'), ('Smoothbeats', 'broadcast.content.genre', 'Contemporary R&B'), ('m.0wjhc6c', 'award.award_honor.award', 'Teen Choice Award for Choice Twitter Personality'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Frank Sinatra', 'people.person.profession', 'Actor'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Estelle', 'people.person.gender', 'Female'), ('Parents', 'type.property.schema', 'Person'), ('Justin Bieber: Never Say Never', 'film.film.sequel', \"Justin Bieber's Believe\"), ('Never Let You Go', 'music.composition.recordings', 'Never Let You Go'), ('#Thatpower', 'music.recording.artist', 'Justin Bieber'), ('WildFMRadio.com', 'broadcast.content.genre', 'Rap music'), ('Carrie Underwood', 'broadcast.artist.content', 'radioIO Todays POP'), ('Van Halen', 'broadcast.artist.content', 'Sunshine Radio'), ('Lady Gaga', 'broadcast.artist.content', 'Hot Wired Radio'), ('Baby', 'music.composition.recordings', 'Baby'), ('Mary J. Blige', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Pattie Mallette', 'book.author.works_written', \"Nowhere but Up: The Story of Justin Bieber's Mom\"), ('m.0_x4zg3', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gc_9w6', 'common.webpage.topic', 'Justin Bieber'), ('Pray', 'music.single.versions', 'Baby (acoustic)'), ('Ice Cube', 'people.person.languages', 'English Language'), ('Runaway Love (remix)', 'music.recording.tracks', 'Runaway Love (remix)'), ('Kylie Minogue', 'broadcast.artist.content', 'Sunshine Radio'), ('1.FM Top 40', 'broadcast.content.artist', 'Wayne Wonder'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('As Long As You Love Me (Audien Luvstep mix)', 'common.topic.notable_types', 'Musical Recording'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Massimo Boldi'), ('Soulja Boy', 'people.person.place_of_birth', 'Chicago'), ('Ciara', 'broadcast.artist.content', '1Club.FM: Power'), ('Taylor Swift', 'music.artist.genre', 'Pop music'), ('PowerHitz', 'broadcast.content.artist', 'Sean Paul'), ('PowerHitz', 'broadcast.content.artist', 'Fabolous'), ('Lil Jon', 'people.person.profession', 'Actor'), ('All That Matters', 'music.album.releases', 'All That Matters'), ('Pray', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'Somebody to Love'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Paramore'), ('Duffy', 'people.person.profession', 'Singer-songwriter'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO RNB Mix'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Usher'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Usher'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Avi Youabian'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_vkd'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0z340zt'), ('Dance music', 'broadcast.genre.content', 'Hot 97.7'), ('Musician', 'common.topic.notable_types', 'Profession'), ('Yves Bole', 'base.musicpf.home_page.artist', 'Yves Bole'), ('Synthpop', 'common.topic.notable_types', 'Musical genre'), ('1Club.FM: V101', 'broadcast.content.artist', 'Usher'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Children'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h521z'), ('Contemporary R&B', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('Adrienne Bailon', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftks'), ('Taylor Swift', 'music.artist.genre', 'Rock music'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Indie rock', 'common.topic.subjects', 'Indie rock'), ('Kelis', 'people.person.profession', 'Singer-songwriter'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Never Say Never (acoustic)', 'common.topic.notable_types', 'Musical Recording'), ('Nelly Furtado', 'people.person.profession', 'Actor'), ('My World', 'music.album.genre', 'Pop music'), ('Justin Bieber: Never Say Never', 'common.topic.notable_types', 'Award-Winning Work'), ('All Around the World', 'music.album.primary_release', 'All Around the World'), ('Boyfriend (acoustic version)', 'music.recording.song', 'Boyfriend'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Will i Am', 'award.award_nominee.award_nominations', 'm.0y84ynj'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m38'), ('m.0_w1gn3', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0njvvj_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Twista'), ('Christina Milian', 'music.artist.label', 'Island Records'), ('Gwen Stefani', 'people.person.gender', 'Female'), ('2010 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw1tn'), ('P!nk', 'broadcast.artist.content', 'Hot Wired Radio'), ('Jordin Sparks', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('P!nk', 'people.person.nationality', 'United States of America'), ('All Bad', 'music.album.release_type', 'Single'), ('Sean Kingston', 'common.topic.notable_types', 'Musical Artist'), ('K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Hot Wired Radio', 'broadcast.content.artist', 'KT Tunstall'), ('Eminem', 'broadcast.artist.content', '1Club.FM: Channel One'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft0d'), ('Gavin DeGraw', 'people.person.profession', 'Singer'), ('Jessie J', 'common.topic.notable_types', 'Musical Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Brandy Norwood'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0v90skf'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Daniel Bedingfield', 'music.artist.genre', 'Dance-pop'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Teyana'), ('Foreign Remix', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.episode', 'Finalists Chosen #1'), ('Believe Tour', 'common.topic.notable_for', 'g.1jffqwjl0'), ('The Roots', 'broadcast.artist.content', 'Smoothbeats'), ('Justin Bieber: Rise to Fame', 'film.film.personal_appearances', 'm.0nh04xx'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Christina Milian'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('m.0yrjynf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('The Isley Brothers', 'music.artist.genre', 'Quiet Storm'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('MTV Movie Award for Best Jaw Dropping Moment', 'award.award_category.winners', 'm.0pc670l'), ('Tricky Stewart', 'people.person.nationality', 'United States of America'), ('1.FM Top 40', 'broadcast.content.artist', 'Evanescence'), ('Everlast', 'music.artist.label', 'The Island Def Jam Music Group'), ('Change Me', 'music.album.artist', 'Justin Bieber'), ('Pop music', 'common.topic.notable_types', 'Musical genre'), ('Ice Cube', 'broadcast.artist.content', 'Smoothbeats'), ('As Long as You Love Me', 'music.composition.form', 'Song'), ('The Black Eyed Peas', 'broadcast.artist.content', 'FLOW 103'), ('Lil Wayne', 'music.artist.genre', 'Rhythm and blues'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Never Say Never', 'music.album.primary_release', 'Never Say Never'), ('1.FM Top 40', 'broadcast.content.artist', 'Goo Goo Dolls'), ('Singer-songwriter', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('End Date', 'rdf-schema#range', 'Date/Time'), ('Confident', 'music.composition.composer', 'Verse Simmonds'), ('Beautiful', 'music.single.versions', 'Beautiful'), ('Shaggy', 'music.artist.genre', 'World music'), ('Clay Aiken', 'music.artist.genre', 'Pop music'), ('Kelly Clarkson', 'broadcast.artist.content', '.977 The Hits Channel'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'music.recording.artist', 'Carly Rae Jepsen'), (\"1Club.FM: Jammin' Oldies\", 'common.topic.image', '1clubfm.jpg'), ('Hot Wired Radio', 'common.topic.image', 'c.jpg'), ('m.0vp755b', 'music.recording_contribution.contributor', 'Nicki Minaj'), ('My Worlds', 'music.album.artist', 'Justin Bieber'), ('Terius Nash', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Shaffer Smith', 'broadcast.artist.content', '.977 The Hits Channel'), ('J. Holiday', 'people.person.gender', 'Male'), ('Boyfriend', 'music.album.primary_release', 'Boyfriend'), ('J. Holiday', 'people.person.nationality', 'United States of America'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ja Rule'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Anastacia', 'music.artist.genre', 'Pop music'), ('Bigger', 'music.composition.composer', 'Kevin Risto'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhhqv'), ('Rihanna', 'freebase.valuenotation.has_no_value', 'Children'), ('Sean Kingston', 'people.person.nationality', 'United States of America'), ('m.0h17ns7', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('m.0ghz3d6', 'tv.tv_guest_role.episodes_appeared_in', 'Cheryl Cole and Alicia Keys @ R1BW'), ('Anastacia', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('JellyRadio.com', 'broadcast.content.artist', 'Ice Cube'), ('Hold Tight', 'music.album.primary_release', 'Hold Tight'), ('m.0t4s_bn', 'award.award_honor.ceremony', 'Juno Awards of 2013'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Big R Radio - The Hawk', 'common.topic.notable_for', 'g.125500lwf'), ('m.0_svs8m', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Never Let You Go', 'common.topic.notable_types', 'Canonical Version'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.composition.recordings', 'Baby'), ('Sheryl Crow', 'music.artist.genre', 'Rock music'), ('Jessica Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Tommy Sands'), ('Contemporary R&B', 'broadcast.genre.content', '181-party'), ('m.0gz0fr6', 'film.personal_film_appearance.person', 'Pattie Mallette'), ('m.0v90skf', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('m.09xh0kp', 'common.webpage.topic', 'Record producer'), ('Janet Jackson', 'people.person.profession', 'Actor'), ('Jennifer Lopez', 'people.person.profession', 'Film Producer'), ('Janet Jackson', 'broadcast.artist.content', 'Sunshine Radio'), ('Tommy Sands', 'people.person.profession', 'Actor'), ('m.0y84ynj', 'award.award_nomination.nominated_for', '#thatPower'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chris Brown', 'music.artist.genre', 'Dance-pop'), ('PowerHitz', 'broadcast.content.artist', 'Ciara'), ('181-beat', 'broadcast.content.artist', 'Sean Combs'), ('Hold Tight', 'music.album.artist', 'Justin Bieber'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Lupe Fiasco', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Elvis Presley', 'people.person.profession', 'Actor'), ('Jordin Sparks', 'people.person.profession', 'Actor'), ('Yves Bole', 'base.fashionmodels.fashion_model.eye_color', 'Brown'), ('Tyga', 'people.person.profession', 'Actor'), ('m.0bnstws', 'common.webpage.in_index', 'Blissful Master Index'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelly Clarkson', 'people.person.profession', 'Singer-songwriter'), ('Big Sean', 'music.artist.genre', 'Hip hop music'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Music executive', 'people.profession.specializations', 'Record producer'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Sia Furler', 'music.artist.genre', 'Indie rock'), ('m.0sgkyfg', 'award.award_honor.award_winner', 'Justin Bieber'), ('Next to You', 'music.album.compositions', 'Next to You'), ('Usher', 'broadcast.artist.content', '181-beat'), ('Nicki Minaj', 'music.featured_artist.albums', 'Beauty and a Beat (Remixes)'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Chingy', 'broadcast.artist.content', 'PowerHitz'), ('Bigger', 'music.composition.composer', 'Dapo Torimiro'), (\"Justin Bieber's Believe\", 'film.film.genre', 'Music'), ('Britney Spears', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Lady Antebellum', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), (\"Justin Bieber's Believe\", 'film.film.language', 'English Language'), ('Avril Lavigne', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('.977 The Hits Channel', 'broadcast.content.artist', \"Destiny's Child\"), ('Paul Anka', 'people.person.nationality', 'Canada'), ('Hikaru Utada', 'music.artist.genre', 'Pop music'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All That Matters', 'music.recording.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Frankie J'), ('Next to You', 'music.recording.canonical_version', 'Next to You'), ('Jordan Francis', 'people.person.profession', 'Dancer'), ('Lupe Fiasco', 'people.person.gender', 'Male'), ('Bryan Adams', 'music.artist.genre', 'Rock music'), ('Paul Anka', 'base.icons.icon.icon_genre', 'Teen idol'), ('Jaden Smith', 'music.featured_artist.albums', 'Never Say Never'), ('Beautiful', 'music.recording.canonical_version', 'Beautiful'), ('Van Halen', 'music.artist.genre', 'Rock music'), ('JoJo', 'music.artist.genre', 'Pop music'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('St. Michael Catholic Secondary School', 'education.educational_institution.students_graduates', 'm.0w5l5h1'), ('Bryan Adams', 'people.person.nationality', 'Canada'), ('Janet Jackson', 'people.person.profession', 'Singer-songwriter'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010n0hfx'), ('All Around The World (featuring Ludacris)', 'common.topic.notable_types', 'Musical Recording'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('Kelly Clarkson', 'people.person.profession', 'Record producer'), ('Person', 'type.type.properties', 'Date of birth'), ('Will Smith', 'people.person.children', 'Jaden Smith'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Profession'), ('NME Award for Worst Dressed', 'award.award_category.winners', 'm.0z8s562'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)'), ('HitzRadio.com', 'broadcast.content.artist', '50 Cent'), ('Colbie Caillat', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('NME Awards 2012', 'award.award_ceremony.awards_presented', 'm.0z83x9l'), ('Hot Wired Radio', 'broadcast.content.genre', 'Pop music'), ('Live My Life', 'common.topic.notable_for', 'g.11b75k11dz'), ('School Boy Records', 'common.topic.notable_for', 'g.1258hxrpm'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('HitzRadio.com', 'broadcast.content.artist', 'Shaggy'), ('Weblink', 'rdf-schema#domain', 'Topic'), ('Justin Bieber', 'base.popstra.celebrity.infidelity_victim', 'm.0gxnp2l'), ('Stratford', 'location.location.containedby', 'Ontario'), ('Avril Lavigne', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Pop music'), ('Mary J. Blige', 'people.person.profession', 'Model'), ('Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Stephen Melton', 'people.person.profession', 'Musician'), ('m.0ws63cr', 'award.award_nomination.award_nominee', 'The Island Def Jam Music Group'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Gas Pedal', 'music.recording.song', 'Gas Pedal'), ('Recovery', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('m.0102z0vx', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kelis', 'music.artist.genre', 'Hip hop music'), ('Lil Wayne', 'people.person.profession', 'Record producer'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('DMX', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Power'), ('FLOW 103', 'broadcast.content.artist', 'Tupac Shakur'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lil Wayne'), ('Demi Lovato', 'people.person.profession', 'Actor'), ('Justin Timberlake', 'common.topic.notable_types', 'Musical Artist'), ('1.FM Top 40', 'broadcast.content.artist', 'Simple Plan'), ('Europop', 'common.topic.notable_types', 'Musical genre'), ('Verse Simmonds', 'common.topic.notable_types', 'Musical Artist'), ('Never Say Never', 'music.recording.artist', 'Justin Bieber'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Mariah Carey', 'people.person.nationality', 'United States of America'), ('Right Here (featuring Drake)', 'music.recording.releases', 'Believe'), ('m.07053l5', 'common.webpage.resource', 'm.0bm1zq9'), ('Bad Day', 'music.composition.composer', 'Ronald Isley'), ('David Cassidy', 'people.person.profession', 'Actor'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Rob Thomas', 'freebase.valuenotation.has_value', 'Parents'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Kelly Clarkson'), ('m.0101ft1r', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Scooter Braun'), ('JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw59_'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkc0l'), ('Tommy Sands', 'people.person.gender', 'Male'), ('m.0n58kgb', 'award.award_nomination.ceremony', 'American Music Awards of 2012'), ('Duffy', 'music.artist.genre', 'Pop music'), ('Taylor Swift', 'broadcast.artist.content', '1.FM Top 40'), ('Gender', 'rdf-schema#domain', 'Person'), ('Donna Summer', 'people.person.profession', 'Musician'), ('Ludacris', 'music.composer.compositions', 'All Around The World'), ('Dancer', 'people.profession.specialization_of', 'Artist'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010n0htg'), ('Die in Your Arms', 'music.single.versions', 'Die in Your Arms'), ('Miley Cyrus', 'music.artist.genre', 'Pop music'), ('Usher', 'people.person.gender', 'Male'), ('1Club.FM: Power', 'broadcast.content.artist', 'Lil Jon'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Rehab'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Never Say Never', 'music.album.release_type', 'Single'), ('m.0r90dwg', 'common.webpage.category', 'Topic Webpage'), ('C1', 'people.person.nationality', 'United States of America'), ('Canadian', 'people.ethnicity.includes_groups', 'Scottish Canadian'), ('Justin Bieber: Never Say Never', 'film.film.language', 'English Language'), ('Amerie', 'common.topic.notable_types', 'Musical Artist'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Keyshia Cole', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Mann', 'people.person.nationality', 'United States of America'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('K-Ci & JoJo', 'broadcast.artist.content', '1Club.FM: Power'), ('Film Producer', 'common.topic.notable_types', 'Profession'), ('RedOne', 'music.artist.genre', 'Contemporary R&B'), ('Chloe Bridges', 'people.person.profession', 'Actor'), ('Juelz Santana', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Indie rock', 'common.topic.subject_of', 'Pop music'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The Fray'), ('Annette Funicello', 'people.person.gender', 'Female'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Tupac Shakur'), ('Britney Spears', 'music.artist.genre', 'Dance music'), ('Lil Wayne', 'people.person.languages', 'English Language'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Baby Bash'), ('Teen idol', 'common.topic.webpage', 'm.09y8cm9'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0101fvfh', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Mariah Carey', 'common.topic.notable_types', 'Musical Artist'), ('Rhythm and blues', 'music.genre.subgenre', 'K-pop'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.composer.compositions', 'First Dance'), ('2011 MTV Movie Awards', 'time.event.locations', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Avril Lavigne', 'people.person.nationality', 'Canada'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('m.0sgkw8v', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('Mary J. Blige', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0101ftrz', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Die in Your Arms'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Kerrang! Awards 2012', 'award.award_ceremony.awards_presented', 'm.0y7y53r'), ('Marvin Isley', 'freebase.valuenotation.has_value', 'Parents'), ('FLOW 103', 'broadcast.content.artist', 'Mary J. Blige'), ('Chingy', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'music.composer.compositions', 'Thought Of You'), ('Paul Anka', 'people.person.profession', 'Actor'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('JellyRadio.com', 'broadcast.content.genre', 'Reggae'), ('Ellen DeGeneres', 'people.person.profession', 'Actor'), ('P!nk', 'people.person.gender', 'Female'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Terius Nash', 'people.person.gender', 'Male'), ('m.0ywsnc9', 'award.award_nomination.ceremony', '5th Annual Shorty Awards'), ('Bryan Adams', 'common.topic.notable_types', 'Musical Artist'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.episode', 'Top 3 Finalists Perform'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0njhyh_', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Juvenile'), ('Tommy Sands', 'people.person.nationality', 'United States of America'), ('Kanye West', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jon M. Chu', 'people.person.nationality', 'United States of America'), ('Katy Perry', 'people.person.nationality', 'United States of America'), ('Lil Wayne', 'music.artist.genre', 'Pop music'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3_7'), ('Chris Brown', 'music.artist.genre', 'Contemporary R&B'), ('A Star Was Born: Justin Bieber', 'film.film.genre', 'Music'), ('Nelly', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Jules & Raoul'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkyfg'), ('Film', 'freebase.type_hints.included_types', 'Topic'), ('Jennifer Lopez', 'music.artist.label', 'The Island Def Jam Music Group'), ('Award-Winning Work', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Instrumental)'), ('m.0y5tj13', 'film.personal_film_appearance.film', '30 Days in May'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('Pras', 'broadcast.artist.content', '.977 The Hits Channel'), ('Weblink', 'type.property.schema', 'Topic'), ('Roller Coaster', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Bad Day', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z340zt'), ('Runaway Love', 'music.recording.song', 'Runaway Love'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ludacris', 'award.award_winner.awards_won', 'm.0n1ykxp'), ('Brandy Norwood', 'people.person.nationality', 'United States of America'), ('Kid Cudi', 'music.artist.genre', 'Rock music'), ('Boyfriend (Neon NiteClub Remix)', 'common.topic.notable_types', 'Musical Recording'), ('Pras', 'people.person.profession', 'Film Producer'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.location', 'Chicago'), ('P!nk', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc3_1'), ('Coldplay', 'broadcast.artist.content', 'radioIO Todays POP'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('m.0hvlt03', 'film.film_film_distributor_relationship.distributor', 'Paramount Pictures'), ('Teen pop', 'common.topic.notable_for', 'g.1259fbz0c'), ('Fabolous', 'people.person.profession', 'Actor'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Outkast'), ('Aaliyah', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Fergie', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.genre', 'Dance music'), ('Rihanna', 'music.artist.genre', 'Synthpop'), ('Ashley Tisdale', 'people.person.profession', 'Film Producer'), ('Michael Jackson', 'music.artist.genre', 'Pop music'), ('Juno Award for Pop Album of the Year', 'award.award_category.winners', 'm.0njvtth'), ('Mary J. Blige', 'music.artist.genre', 'Hip hop music'), ('Kid Cudi', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Vanessa Hudgens', 'people.person.profession', 'Singer'), ('m.0z1scxk', 'award.award_honor.award_winner', 'Justin Bieber'), ('Madonna', 'people.person.nationality', 'United States of America'), ('Juicy J', 'common.topic.notable_types', 'Musical Artist'), ('Ashley Tisdale', 'broadcast.artist.content', 'Hot Wired Radio'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ultrabeat vs. Darren Styles'), ('Under the Mistletoe', 'common.topic.article', 'm.0h7l5vh'), ('Beautiful', 'music.recording.tracks', 'g.11b81rhqyh'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Madonna'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Shaffer Smith'), ('Hikaru Utada', 'music.artist.genre', 'Rhythm and blues'), ('Mary J. Blige', 'people.person.gender', 'Female'), ('Anastacia', 'people.person.profession', 'Dancer'), ('Runaway Love (remix)', 'music.album.featured_artists', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvmb'), ('m.0dj34q5', 'common.webpage.resource', 'Justin Bieber Fan Club'), ('Katy Perry: Part of Me', 'film.film.rating', 'PG (USA)'), ('justinbieber', 'internet.blog.language', 'English Language'), ('Zac Brown Band', 'broadcast.artist.content', 'Hot Wired Radio'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0jvgmxc'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Die in Your Arms', 'music.composition.composer', 'Rodney Jerkins'), ('HitzRadio.com', 'broadcast.content.artist', 'New Kids on the Block'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('Tampa', 'base.biblioness.bibs_location.country', 'United States of America'), ('Chingy', 'music.artist.genre', 'Hip hop music'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.featured_artists', 'Big Sean'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Paramore'), ('Pray', 'music.composition.language', 'English Language'), ('Blackstreet', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108j492'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Lolly', 'music.recording.artist', 'Maejor'), ('m.0njw4z2', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('PowerHitz', 'broadcast.content.genre', 'Hip hop music'), ('My World', 'music.album.primary_release', 'My World'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fs_z'), ('Whitney Houston', 'people.person.profession', 'Singer'), ('Nelly', 'common.topic.notable_types', 'Musical Artist'), ('Britney Spears', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'people.person.education', 'm.0yrlqp1'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Snoop Dogg'), ('Stuart Ford', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Never Say Never'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Madonna'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Dub Mix)'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k1h'), ('Baby', 'music.composition.composer', 'Justin Bieber'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Profession'), ('iJustine', 'influence.influence_node.influenced', 'Yves Bole'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Big K.R.I.T.'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Brutha'), ('1Club.FM: V101', 'broadcast.content.artist', 'Mary J. Blige'), ('All That Matters', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Max Martin', 'music.artist.genre', 'Contemporary R&B'), ('Ray J', 'freebase.valuenotation.has_no_value', 'Children'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Chris Brown'), ('Kelis', 'common.topic.notable_types', 'Musical Artist'), ('Whitney Houston', 'music.artist.genre', 'Contemporary R&B'), ('m.09y8dts', 'common.webpage.topic', 'Teen idol'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft5f'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Never Say Never', 'common.topic.notable_for', 'g.12555vkyk'), ('Beyoncé Knowles', 'music.artist.genre', 'Dance music'), ('Never Say Never', 'music.composition.composer', 'Justin Bieber'), ('Jaden Smith', 'people.person.profession', 'Singer-songwriter'), ('Jordan Pruitt', 'common.topic.notable_types', 'Musical Artist'), ('Under the Mistletoe', 'music.album.genre', 'Dance-pop'), ('Ciara', 'music.artist.genre', 'Dance music'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All Around the World', 'music.album.artist', 'Justin Bieber'), ('Jane Lipsitz', 'people.person.gender', 'Female'), ('Mann', 'music.artist.label', 'The Island Def Jam Music Group'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bigger', 'freebase.valuenotation.is_reviewed', 'Lyricist'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Lil Jon', 'people.person.gender', 'Male'), ('m.0j8z6tl', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('m.0njw444', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0101fv4x', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Die in Your Arms', 'music.composition.recordings', 'Die in Your Arms'), ('Musician', 'people.profession.specializations', 'Chansonnier'), ('Adrienne Bailon', 'music.artist.genre', 'Dance music'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('World music', 'music.genre.subgenre', 'Pop music'), ('m.010lkp2z', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('m.09wnjlc', 'common.webpage.topic', 'Teen idol'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Singer-songwriter', 'people.profession.specializations', 'Chansonnier'), ('m.09wf9d1', 'common.webpage.topic', 'Teen idol'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.song', 'As Long as You Love Me'), ('Kevin Risto', 'people.person.nationality', 'Canada'), ('Jazmyn Bieber', 'people.person.place_of_birth', 'Canada'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3dn'), ('Bad Day', 'common.topic.notable_types', 'Composition'), ('HitzRadio.com', 'broadcast.content.artist', 'Big Pun'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('GotRadio - RnB Classics', 'broadcast.content.genre', 'Top 40'), ('Rihanna', 'broadcast.artist.content', '1.FM Top 40'), ('All Around The World', 'music.composition.recordings', 'All Around the World (acoustic version)'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten radio)'), ('Amerie', 'people.person.profession', 'Actor'), ('JellyRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0vpggkk', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Athan Grace', 'people.person.profession', 'Actor'), ('m.0n1ykxp', 'award.award_honor.ceremony', '2010 MTV Video Music Awards'), ('Stuart Ford', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjy9'), ('Pop music', 'common.topic.subjects', 'Indie rock'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Album'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('JellyRadio.com', 'broadcast.content.genre', 'Dance music'), ('m.0gxnp26', 'base.popstra.breakup.participant', 'Justin Bieber'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Canada', 'common.topic.notable_types', 'Country'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Avril Lavigne', 'people.person.gender', 'Female'), ('Lady Gaga', 'music.artist.genre', 'Rock music'), ('m.0_vw6nn', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Jason Mraz'), ('Thought Of You', 'music.recording.song', 'Thought Of You'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw1tn'), ('Sia Furler', 'people.person.gender', 'Female'), ('Kylie Minogue', 'people.person.profession', 'Singer'), (\"Destiny's Child\", 'common.topic.notable_types', 'Musical Artist'), ('Avril Lavigne', 'common.topic.notable_types', 'Musical Artist'), ('Rihanna', 'broadcast.artist.content', '181-beat'), ('Beauty and a Beat (Remixes)', 'music.album.releases', 'Beauty and a Beat (Remixes)'), ('Christina Milian', 'music.artist.label', 'The Island Def Jam Music Group'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Nasri', 'music.artist.genre', 'Rhythm and blues'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelis', 'music.artist.genre', 'Synthpop'), ('1Club.FM: Power', 'broadcast.content.artist', 'Ciara'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Kate Nash'), ('Stratford', 'location.location.people_born_here', 'Pattie Mallette'), ('Sean Kingston', 'music.composer.compositions', 'Eenie Meenie'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('The Notorious B.I.G.', 'people.person.languages', 'English Language'), ('Justin Bieber', 'music.artist.album', 'U Smile'), ('Right Here', 'music.composition.composer', 'Justin Bieber'), ('Usher', 'broadcast.artist.content', '1Club.FM: V101'), ('City/Town/Village', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'base.schemastaging.person_extra.net_worth', 'm.0j8mhs_'), ('m.0y5tj13', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Singer-songwriter'), ('Profession', 'rdf-schema#range', 'Profession'), ('Shaffer Smith', 'music.artist.genre', 'Pop music'), ('Alanis Morissette', 'people.person.nationality', 'United States of America'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Jeremy Bieber', 'common.topic.notable_types', 'Person or entity appearing in film'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kylie Minogue', 'people.person.profession', 'Musician'), ('Gwen Stefani', 'music.artist.genre', 'Reggae'), ('Max Martin', 'people.person.profession', 'Musician'), ('Justin Bieber', 'music.artist.album', 'PYD'), ('Michael Jackson', 'broadcast.artist.content', 'Classic Soul Network'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('Duffy', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Britney Spears', 'people.person.profession', 'Film Producer'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Robby Stewart'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.genre', 'Documentary film'), ('Caitlin Beadles', 'people.person.gender', 'Female'), ('Person', 'type.type.properties', 'Place of birth'), ('Christina Milian', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Zac Efron', 'people.person.nationality', 'United States of America'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Santana'), ('Eenie Meenie', 'common.topic.notable_types', 'Composition'), ('Somebody to Love (remix)', 'music.album.album_content_type', 'Remix album'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Twista', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsc'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ronski Speed'), ('Hot Wired Radio', 'broadcast.content.artist', 'Fall Out Boy'), ('JellyRadio.com', 'broadcast.content.artist', 'Snoop Dogg'), ('Singer-songwriter', 'people.profession.specialization_of', 'Musician'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 64kbps Stream'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.composition.recordings', 'Baby'), ('Rihanna', 'broadcast.artist.content', 'Sunshine Radio'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cielo Drive'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Parents'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft35'), ('Beautiful and the Beat', 'music.recording.artist', 'Justin Bieber'), ('Aaliyah', 'music.artist.genre', 'Rhythm and blues'), ('Will i Am', 'people.person.profession', 'Record producer'), ('Justin Bieber', 'common.topic.webpage', 'm.0dj34q5'), ('m.0njwqrb', 'freebase.valuenotation.has_no_value', 'Winning work'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('The Black Eyed Peas', 'music.artist.genre', 'Trance music'), ('m.0115qhzk', 'award.award_honor.award', 'MTV Europe Music Voices Award'), ('m.0z898w6', 'award.award_honor.honored_for', 'CSI: Crime Scene Investigation'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0vpggkk', 'music.recording_contribution.album', 'We Are the World 25 for Haiti'), ('Avril Lavigne', 'people.person.profession', 'Actor'), ('Frank Ocean', 'music.artist.genre', 'Contemporary R&B'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Victoria Justice'), ('Runaway Love (remix)', 'music.album.contributor', 'm.0vp7m_x'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'music.single.versions', 'Baby'), ('Trey Songz', 'people.person.gender', 'Male'), ('Where Are Ü Now', 'music.recording.tracks', 'Where Are Ü Now'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Usher', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3cx'), ('m.0115qhzk', 'award.award_honor.award_winner', 'Justin Bieber'), ('Location', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhvsq'), ('Whitney Houston', 'people.person.profession', 'Musician'), ('Miley Cyrus', 'music.artist.genre', 'Teen pop'), ('Yves Bole', 'influence.influence_node.influenced', 'Jessie J'), ('Die in Your Arms', 'music.composition.composer', 'Dennis \\\\\"Aganee\\\\\" Jenkins'), ('Pattie Mallette', 'people.person.profession', 'Film Producer'), ('m.0102z0vx', 'award.award_honor.award_winner', 'Justin Bieber'), ('LeAnn Rimes', 'people.person.gender', 'Female'), ('Gas Pedal', 'music.recording.artist', 'Sage The Gemini'), ('Rob Thomas', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Gavin DeGraw', 'people.person.profession', 'Actor'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Place of birth'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Britney Spears'), ('#thatPOWER', 'music.album.contributor', 'm.0vpgdpb'), ('Jessica Simpson', 'people.person.profession', 'Singer'), ('Lady Gaga', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juicy J', 'people.person.nationality', 'United States of America'), ('Big R Radio - The Hawk', 'common.topic.notable_types', 'Broadcast Content'), ('Trick Daddy', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'people.person.nationality', 'United States of America'), ('m.0gd28hv', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Sheryl Crow', 'people.person.profession', 'Artist'), (\"Destiny's Child\", 'broadcast.artist.content', '1Club.FM: Mix 106'), ('A Star Was Born: Justin Bieber', 'film.film.genre', 'Documentary film'), ('Ricky Nelson', 'common.topic.notable_types', 'Musical Artist'), ('Ray J', 'people.person.profession', 'Record producer'), ('Italian Language', 'language.human_language.main_country', 'Italy'), ('Baby', 'music.album.artist', 'Justin Bieber'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Profession'), ('American Music Award for New Artist of the Year', 'award.award_category.winners', 'm.0zbf_4g'), ('Enrique Iglesias', 'people.person.languages', 'Spanish Language'), ('Never Say Never (acoustic)', 'music.recording.featured_artists', 'Jaden Smith'), ('Geri Halliwell', 'people.person.profession', 'Singer'), ('Justin Bieber: Just Getting Started', 'book.translated_work.translations', 'Justin Bieber : bara början'), ('HitzRadio.com', 'broadcast.content.artist', 'Sisqó'), ('181-beat', 'broadcast.content.genre', 'Pop music'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'C1'), ('Musical Artist', 'common.topic.article', 'm.01z0mrz'), ('Gas Pedal', 'music.single.versions', 'Gas Pedal'), ('m.0v_714c', 'film.personal_film_appearance.person', 'Justin Bieber'), ('BeirutNights.com Radio', 'common.topic.article', 'm.03hrx6z'), ('m.0t4syfh', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('United States of America', 'base.biblioness.bibs_topic.subsumes', 'United States of America'), ('Nasri', 'people.person.profession', 'Record producer'), ('Donna Summer', 'music.artist.genre', 'Rock music'), ('Baby (acoustic)', 'music.recording.artist', 'Justin Bieber'), ('Somebody to Love', 'common.topic.notable_for', 'g.1yl5kzcrs'), ('Avril Lavigne', 'people.person.languages', 'English Language'), ('m.0d33hsc', 'celebrities.friendship.friend', 'Justin Bieber'), ('Michael Jackson', 'music.artist.genre', 'Rhythm and blues'), ('Kelis', 'music.artist.genre', 'Rhythm and blues'), ('Alanis Morissette', 'broadcast.artist.content', '1.FM Top 40'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('Kanye West', 'music.artist.album', 'Runaway Love (remix)'), ('Hot Wired Radio', 'broadcast.content.genre', 'Urban contemporary'), ('Mariah Carey', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Montell Jordan', 'people.person.gender', 'Male'), ('One Less Lonely Girl', 'common.topic.notable_types', 'Musical Album'), ('Usher', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0gbm3d_', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Never Say Never', 'music.composition.composer', 'Adam Messinger'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'MC Ren'), ('m.0tkc3tj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0101ft3d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Italian Language', 'language.human_language.countries_spoken_in', 'Italy'), ('Die in Your Arms', 'music.composition.composer', 'Travis Sayles'), ('Aaliyah', 'common.topic.notable_types', 'Musical Artist'), ('1.FM Top 40', 'broadcast.content.artist', 'Uncle Kracker'), ('Baby', 'music.composition.composer', 'Tricky Stewart'), ('Eenie Meenie', 'music.composition.composer', 'Sean Kingston'), ('My World 2.0', 'music.album.album_content_type', 'Studio album'), ('Sheryl Crow', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'music.artist.label', 'RBMG Records'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Santana', 'broadcast.artist.content', 'radioIO Classic RNB'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Adrienne Bailon', 'music.artist.genre', 'Hip hop music'), ('Rihanna', 'people.person.gender', 'Female'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Justin Bieber'), ('Willa Ford', 'people.person.profession', 'Record producer'), ('Red Hot Chili Peppers', 'broadcast.artist.content', 'Hot Wired Radio'), ('L.A. Reid', 'people.person.profession', 'Music executive'), ('Timbaland', 'people.person.profession', 'Musician'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvkw'), ('Redfoo', 'music.artist.genre', 'Hip hop music'), ('Shaggy', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Pras', 'music.artist.genre', 'Hip hop music'), ('Ice Cube', 'people.person.gender', 'Male'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never (acoustic)'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Believe', 'music.album.releases', 'Believe'), ('The Island Def Jam Music Group', 'common.topic.article', 'm.04fc6k'), ('Adam Messinger', 'people.person.profession', 'Record producer'), ('Daniel Bedingfield', 'music.artist.genre', 'Pop music'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.episode', 'The Father-In-Law'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Stratford', 'location.location.people_born_here', 'Jeremy Bieber'), ('Athan Grace', 'people.person.nationality', 'United States of America'), ('Lolly', 'music.composition.composer', 'Justin Bieber'), ('m.0z340zt', 'award.award_honor.award', 'MTV Europe Music Award for Best World Stage Performance'), ('Timbaland', 'music.artist.genre', 'Rock music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Justin Bieber'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'music.artist.genre', 'Rhythm and blues'), ('Spouse', 'owl#inverseOf', 'Spouse (or domestic partner)'), ('My Worlds', 'music.album.releases', 'My Worlds'), ('Kuk Harrell', 'music.artist.genre', 'Hip hop music'), ('The Notorious B.I.G.', 'people.person.profession', 'Record producer'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Award category'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Carrie Underwood'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Daniel Bedingfield'), ('J. Holiday', 'people.person.profession', 'Actor'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat'), ('Tyga', 'music.artist.album', 'Wait For a Minute'), ('Katy Perry: Part of Me', 'film.film.executive_produced_by', 'Randy Phillips'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Thought of You', 'music.recording.canonical_version', 'Thought of You'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Enrique Iglesias', 'music.artist.genre', 'Pop music'), ('m.0129jzth', 'people.sibling_relationship.sibling', 'Yves Bole'), ('Kelly Clarkson', 'broadcast.artist.content', 'HitzRadio.com'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Eenie Meenie', 'music.album.artist', 'Justin Bieber'), ('Britney Spears', 'music.artist.genre', 'Urban contemporary'), ('Timbaland', 'music.artist.genre', 'Dance-pop'), ('Contemporary R&B', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('m.0sgkw8v', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jessica Simpson', 'music.artist.genre', 'Dance-pop'), ('radioIO Todays POP', 'common.topic.image', 'RadioIO.png'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr28j'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Children'), ('Brandy Norwood', 'music.artist.genre', 'Hip hop music'), ('The Pussycat Dolls', 'broadcast.artist.content', '.977 The Hits Channel'), ('Heartbreaker', 'common.topic.notable_types', 'Canonical Version'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.song', 'Beauty And A Beat'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chance the Rapper', 'music.composer.compositions', 'Confident'), ('Carrie Underwood', 'music.artist.genre', 'Pop music'), ('Aaliyah', 'people.person.profession', 'Dancer'), ('My World 2.0', 'common.topic.notable_for', 'g.12559ncd9'), ('.977 The Hits Channel', 'broadcast.content.genre', 'Rock music'), ('Blu Cantrell', 'common.topic.notable_types', 'Musical Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The All-American Rejects'), ('My World 2.0', 'music.album.releases', 'My World'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.featured_artists', 'Big Sean'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgk_cw'), ('1.FM Top 40', 'broadcast.content.genre', 'Dance music'), ('Jason Mraz', 'people.person.gender', 'Male'), ('1Club.FM: Power', 'broadcast.content.artist', 'Lady Gaga'), ('m.0zbf_4g', 'award.award_honor.award', 'American Music Award for New Artist of the Year'), ('First Dance', 'common.topic.notable_for', 'g.1z2sqd4jr'), ('Duffy', 'common.topic.notable_types', 'Musical Artist'), ('Chance the Rapper', 'people.person.profession', 'Singer-songwriter'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jennifer Lopez', 'music.artist.genre', 'Dance music'), ('Janet Jackson', 'people.person.nationality', 'United States of America'), ('m.010lkp2z', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Chingy', 'people.person.nationality', 'United States of America'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', \"00's\"), ('Parents', 'type.property.reverse_property', 'Children'), ('The Island Def Jam Music Group', 'music.record_label.artist', '6 Tre G'), ('Nelly Furtado', 'music.artist.genre', 'Dance-pop'), ('Spouse (or domestic partner)', 'type.property.schema', 'Person'), ('Whitney Houston', 'people.person.gender', 'Female'), ('Ray J', 'music.artist.genre', 'Contemporary R&B'), ('m.0gh0fdt', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Justin Bieber', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Michael Jackson', 'people.person.profession', 'Actor'), ('John Mamann', 'people.person.profession', 'Music Producer'), ('m.0pc670l', 'award.award_honor.award_winner', 'Justin Bieber'), ('Sean Kingston', 'broadcast.artist.content', 'Hot Wired Radio'), ('Recovery', 'music.album.primary_release', 'Recovery'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bad Day', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('One Less Lonely Girl', 'music.album.genre', 'Pop music'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Taylor Swift'), ('m.0yrjvlh', 'freebase.valuenotation.has_no_value', 'Winning work'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO Classic RNB'), ('HitzRadio.com', 'broadcast.content.genre', 'Pop music'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Stratford', 'base.wikipedia_infobox.settlement.area_code', 'Area codes 519 and 226'), ('181-beat', 'broadcast.content.artist', 'Outkast'), ('Ricky Nelson', 'people.person.gender', 'Male'), ('Rhythm and blues', 'music.genre.subgenre', 'Pop music'), ('Bryan Adams', 'people.person.gender', 'Male'), ('Never Let You Go', 'music.album.release_type', 'Single'), ('HitzRadio.com', 'broadcast.content.artist', 'Aaliyah'), ('#Thatpower', 'common.topic.notable_for', 'g.12h32d843'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Power'), ('First Dance', 'common.topic.notable_types', 'Musical Recording'), ('P!nk', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'JellyRadio.com'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Paul'), ('Justin Bieber', 'common.topic.notable_types', 'Musical Artist'), ('Will i Am', 'music.artist.genre', 'Pop music'), ('HitzRadio.com', 'broadcast.content.artist', 'Rihanna'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('2011 MTV Video Music Awards', 'award.award_ceremony.awards_presented', 'm.0n4rmg7'), ('All That Matters', 'music.composition.recordings', 'All That Matters'), ('181-beat', 'broadcast.content.genre', 'Hip hop music'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0t4s_bn'), ('m.0njw257', 'award.award_honor.ceremony', '2010 MTV Europe Music Awards'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me (acoustic version)'), ('Linkin Park', 'music.artist.genre', 'Rock music'), ('Heartbreaker', 'common.topic.notable_for', 'g.1ydnq397t'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Nelly'), ('m.0ndc0sf', 'award.award_honor.award_winner', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Kanye West'), ('1Club.FM: Power', 'broadcast.content.artist', 'Justin Timberlake'), ('m.0cq990t', 'common.webpage.category', 'Topic Webpage'), ('m.0z87d3n', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Disco'), ('m.0sxhgqj', 'award.award_nomination.nominated_for', 'Believe'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flobots'), ('Heartbreaker', 'common.topic.notable_types', 'Musical Album'), ('Turn to You (Mother’s Day Dedication)', 'music.album.primary_release', 'Turn to You (Mother’s Day Dedication)'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.song', 'As Long as You Love Me'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.episode', 'Make Your Mark: Shake It Up Dance Off 2012'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'New Kids on the Block'), ('Italy', 'common.topic.notable_types', 'Country'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mr. M.'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Will Smith', 'music.artist.genre', 'Hip hop music'), ('Iggy Azalea', 'music.artist.genre', 'Electronic music'), ('Gavin DeGraw', 'people.person.nationality', 'United States of America'), ('Snoop Dogg', 'broadcast.artist.content', '1Club.FM: Power'), ('Ashlee Simpson', 'music.artist.genre', 'Rock music'), ('Chris Brown', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot remix)'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0101ft2j', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Boyfriend', 'music.album.releases', 'Boyfriend (Dada Life remix)'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Christmas in Washington', 'film.film.language', 'English Language'), ('Hot Wired Radio', 'common.topic.notable_for', 'g.125fs0kff'), ('Leonardo DiCaprio', 'people.person.profession', 'Film Producer'), ('Bad 25', 'film.film.personal_appearances', 'm.0nfnx_3'), ('Sean Combs', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.composer.compositions', 'Baby'), ('Chris Brown', 'broadcast.artist.content', 'radioIO Todays POP'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kid Cudi', 'music.artist.genre', 'Hip hop music'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0_sxby0'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lesley Roy'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvk9'), ('1Club.FM: V101', 'broadcast.content.artist', 'Shaffer Smith'), ('Baby', 'music.music_video.artist', 'Justin Bieber'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc259'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvfh'), ('Home to Mama', 'music.composition.composer', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Christina Milian'), ('End Date', 'type.property.expected_type', 'Date/Time'), ('Teyana', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0vp7qr5', 'music.recording_contribution.album', 'Never Say Never'), ('Somebody to Love (remix)', 'common.topic.notable_for', 'g.126tjdgsh'), ('Hot Wired Radio', 'broadcast.content.artist', 'Rihanna'), ('Justin Bieber', 'music.artist.label', 'School Boy Records'), ('Anastacia', 'people.person.profession', 'Record producer'), ('All Around The World', 'music.composition.lyricist', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Pearl Jam'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0gxnp26', 'base.popstra.breakup.participant', 'Caitlin Beadles'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Children'), ('JoJo', 'people.person.profession', 'Actor'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Power'), ('Never Say Never', 'music.composition.composer', 'Kuk Harrell'), ('Kylie Minogue', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Alicia Keys', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('m.0z3tqqt', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('All Bad', 'music.composition.composer', 'Andre Harris'), ('Britney Spears', 'influence.influence_node.influenced', 'Selena Gomez'), ('Ja Rule', 'people.person.profession', 'Musician'), ('m.0y5tj13', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Benny Blanco', 'music.artist.genre', 'Dance-pop'), ('Taylor Swift', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('m.0z84kwx', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108gyct'), ('Roller Coaster', 'music.composition.composer', 'Justin Bieber'), ('Under the Mistletoe', 'common.topic.image', 'justin-bieber-under-the-mistletoe-album-cover.jpg'), ('PowerHitz', 'broadcast.content.artist', 'Terius Nash'), ('1Club.FM: Power', 'broadcast.content.artist', 'Mariah Carey'), ('Hold Tight', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jll'), ('Donna Summer', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Fergie', 'broadcast.artist.content', 'radioIO Todays POP'), ('P!nk', 'people.person.profession', 'Actor'), ('Yves Bole', 'people.person.languages', 'English Language'), ('Aaliyah', 'music.group_member.instruments_played', 'Vocals'), ('Hip hop music', 'broadcast.genre.content', 'WildFMRadio.com'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkr34'), ('Savan Kotecha', 'people.person.nationality', 'United States of America'), ('Right Here', 'music.single.versions', 'Right Here (featuring Drake)'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Mis-Teeq'), ('Frank Ocean', 'music.lyricist.lyrics_written', 'Bigger'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftq9'), ('Reed Smoot', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Sir Mix-a-Lot', 'people.person.nationality', 'United States of America'), ('Khalil', 'music.artist.label', 'The Island Def Jam Music Group'), ('Emphatic Radio.com!', 'broadcast.content.artist', \"Plain White T's\"), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), ('Justin Bieber', 'people.person.nationality', 'Canada'), ('Madonna', 'broadcast.artist.content', '1.FM Top 40'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Date of birth'), ('Live My Life (Jaywalker remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Nick Jonas'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.artist', 'Sean Kingston'), ('m.0gxnp2l', 'base.popstra.infidelity.victim', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Height'), ('HitzRadio.com', 'broadcast.content.artist', 'Green Day'), ('Stephen Melton', 'music.artist.genre', 'Rhythm and blues'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.episode', 'Season 1, episode 1'), ('Toby Gad', 'music.artist.genre', 'Pop music'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Radio Mix)'), ('Kings of Leon', 'influence.influence_node.influenced_by', 'Pearl Jam'), ('Country of nationality', 'type.property.expected_type', 'Country'), ('All That Matters', 'music.album.primary_release', 'All That Matters'), ('Blackstreet', 'music.artist.genre', 'Pop music'), ('Tricky Stewart', 'music.artist.genre', 'Contemporary R&B'), ('Adam Messinger', 'music.composer.compositions', 'Next to You'), ('Akon', 'people.person.profession', 'Musician'), ('m.0sgkrj4', 'award.award_honor.ceremony', \"2013 Kids' Choice Awards\"), ('.977 The Hits Channel', 'broadcast.content.artist', 'My Chemical Romance'), ('Selena Gomez', 'music.artist.genre', 'Dance music'), ('Daniel Bedingfield', 'people.person.profession', 'Singer-songwriter'), ('Nick Jonas', 'music.artist.genre', 'Pop music'), ('Christina Aguilera', 'people.person.profession', 'Actor'), ('Max Martin', 'people.person.profession', 'Record producer'), ('CSI: Crime Scene Investigation', 'award.award_winning_work.awards_won', 'm.0z898w6'), ('Believe', 'music.album.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Fabian'), ('Janet Jackson', 'broadcast.artist.content', 'radioIO Todays POP'), ('New Kids on the Block', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Brandy Norwood', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z0tmyv'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('Record producer', 'common.topic.notable_types', 'Profession'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhyh_'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.song', 'As Long as You Love Me'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Nelly'), ('Britney Spears', 'people.person.profession', 'Dancer'), ('m.0yrhhqv', 'award.award_honor.award', 'MTV Video Music Brazil Award for Best International Artist'), ('Michael Jackson', 'people.person.languages', 'English Language'), ('Justin Bieber', 'common.topic.webpage', 'm.0d_hbgr'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'common.topic.webpage', 'm.0bnstws'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Brooklyn Bounce'), ('Singer', 'people.profession.specializations', 'Playback Singer'), ('Justin Bieber', 'people.person.profession', 'Singer-songwriter'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Flyleaf'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Somebody to Love (remix)', 'music.album.release_type', 'Single'), ('Anastacia', 'people.person.profession', 'Singer-songwriter'), ('m.0z8755b', 'award.award_honor.award', 'NME Award for Worst Album'), ('Anastacia', 'people.person.profession', 'Singer'), ('m.0bm1zq9', 'common.resource.annotations', 'm.07053l5'), ('Hot Wired Radio', 'broadcast.content.artist', 'Gwen Stefani'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'JoJo'), ('Hikaru Utada', 'people.person.profession', 'Singer'), ('Avril Lavigne', 'people.person.profession', 'Artist'), ('Somebody to Love', 'music.composition.recordings', 'Somebody To Love'), ('1Club.FM: V101', 'broadcast.content.genre', 'Urban contemporary'), ('Hip hop music', 'broadcast.genre.content', 'Smoothbeats'), ('1.FM Top 40', 'broadcast.content.broadcast', '1.FM Top 40 - 32kbps Stream'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Juelz Santana'), ('Chloe Bridges', 'people.person.nationality', 'United States of America'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Pop music'), ('Wait For a Minute', 'music.album.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Justin Bieber'), ('Beauty and a Beat', 'music.album.contributor', 'm.0vp755b'), ('1Club.FM: Power', 'broadcast.content.artist', 'Baby Bash'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Mary J. Blige', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: Power'), ('Soulja Boy', 'people.person.languages', 'English Language'), ('m.0101fvjw', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Montell Jordan', 'freebase.valuenotation.has_value', 'Parents'), ('m.0yrkr34', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber: Never Say Never', 'award.award_nominated_work.award_nominations', 'm.0pc67ly'), ('All That Matters', 'music.recording.song', 'All That Matters'), ('Beyoncé Knowles', 'people.person.nationality', 'United States of America'), ('Demi Lovato', 'people.person.nationality', 'United States of America'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.featured_artist.recordings', 'Wait For a Minute'), ('Madonna', 'people.person.profession', 'Actor'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Janet Jackson', 'people.person.profession', 'Model'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Michael Jackson', 'people.person.profession', 'Film Producer'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Record producer', 'common.topic.subject_of', 'Artfarm'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fabolous', 'freebase.valuenotation.has_value', 'Children'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Kylie Minogue'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jeremy Bieber', 'people.person.gender', 'Male'), ('Beautiful and the Beat', 'music.recording.artist', 'Nicki Minaj'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Love Never Felt So Good', 'common.topic.notable_types', 'Composition'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'André Visior'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc259'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Yves Bole', 'base.musicpf.artist.home_page', 'Yves Bole'), ('Cris Cab', 'music.artist.genre', 'Reggae'), ('Jessica Simpson', 'music.artist.genre', 'Rhythm and blues'), ('Believe', 'music.album.releases', 'Believe'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Miami Beach', 'location.location.containedby', 'United States of America'), ('Timbaland', 'music.artist.genre', 'Hip hop music'), ('Janet Jackson', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('m.0102z0vx', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Justin Bieber', 'music.artist.track_contributions', 'm.0rqp4h0'), ('Estelle', 'music.artist.genre', 'Rhythm and blues'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Sir Mix-a-Lot', 'people.person.languages', 'English Language'), ('Geri Halliwell', 'music.artist.genre', 'Dance music'), ('HitzRadio.com', 'broadcast.content.artist', 'Justin Timberlake'), ('Miley Cyrus', 'people.person.gender', 'Female'), ('Person', 'type.type.properties', 'Height'), ('My World 2.0', 'music.album.primary_release', 'My World 2.0'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('L.A. Reid', 'people.person.profession', 'Film Producer'), ('Ciara', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Electronic dance music', 'music.genre.subgenre', 'Trance music'), ('Juelz Santana', 'people.person.gender', 'Male'), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Geri Halliwell', 'music.artist.genre', 'Dance-pop'), ('WildFMRadio.com', 'broadcast.content.artist', 'Mims'), ('Shaffer Smith', 'people.person.profession', 'Musician'), ('Reggae', 'common.topic.notable_types', 'Musical genre'), ('HitzRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0wjhc6c', 'freebase.valuenotation.has_no_value', 'Winning work'), ('1.FM Top 40', 'broadcast.content.artist', 'Staind'), ('One Time', 'music.album.artist', 'Justin Bieber'), ('Hot Wired Radio', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Judy Garland', 'people.person.profession', 'Actor'), ('Blu Cantrell', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('CL', 'freebase.valuenotation.has_value', 'Parents'), ('Driving under the influence', 'celebrities.reason_for_arrest.celebrities_charged_or_arrested', 'm.0_cz5l3'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelly Clarkson', 'people.person.profession', 'Singer'), ('Pattie Mallette', 'people.person.nationality', 'Canada'), ('Mariah Carey', 'music.artist.genre', 'Contemporary R&B'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ice Cube'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0gwhmhm'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Rossano Prini \\\\\"DJ Ross\\\\\"'), ('Kelis', 'people.person.profession', 'Singer'), ('Ginuwine', 'people.person.nationality', 'United States of America'), ('Confident', 'music.album.releases', 'Confident'), ('m.0z87597', 'award.award_nomination.award', 'NME Award for Worst Album'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrktlv'), ('Donna Summer', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Person', 'type.type.properties', 'Gender'), ('Bryan-Michael Cox', 'music.artist.genre', 'Hip hop music'), ('Change Me', 'common.topic.notable_types', 'Musical Recording'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will i Am', 'music.artist.genre', 'Electronic dance music'), ('m.0z0tgz6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.09jx30f', 'base.sounds.audio_recording_contribution.role', 'Record producer'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jay-Z', 'people.person.languages', 'English Language'), ('Gwen Stefani', 'music.artist.genre', 'Contemporary R&B'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('Record producer', 'people.profession.specialization_of', 'Music Producer'), ('Justin Bieber: Never Say Never', 'film.film.runtime', 'm.0gbm35z'), ('As Long as You Love Me', 'music.recording.releases', 'Believe'), ('KooL CrAzE', 'music.artist.genre', 'Rhythm and blues'), ('Pearl Jam', 'music.artist.origin', 'United States of America'), ('MTV Europe Music Voices Award', 'award.award_category.winners', 'm.0115qhzk'), ('m.0_srv2b', 'award.award_nomination.nominated_for', 'justinbieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Ray J'), ('m.0vp8rhw', 'music.recording_contribution.contributor', 'Nicki Minaj'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhrwc'), ('#thatPower', 'music.recording.canonical_version', '#thatPOWER'), ('Rita Ora', 'music.artist.genre', 'Dance music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Klubbdriver'), ('The Isley Brothers', 'broadcast.artist.content', 'GotRadio - RnB Classics'), ('Person or entity appearing in film', 'freebase.type_hints.included_types', 'Topic'), ('Usher', 'music.artist.genre', 'Dance music'), ('Kings of Leon', 'music.artist.label', 'Island Records'), ('1Club.FM: Power', 'broadcast.content.artist', 'Tupac Shakur'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njwqrb'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Die in Your Arms', 'music.album.primary_release', 'Die in Your Arms'), ('1.FM Top 40', 'broadcast.content.artist', 'Boomtang Boys'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Cherish'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Peter Bjorn and John'), ('Record producer', 'common.topic.subjects', 'Steve Blevins'), ('Urban contemporary', 'broadcast.genre.content', '1Club.FM: V101'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Justin Timberlake'), ('Aaliyah', 'people.person.profession', 'Singer'), ('HitzRadio.com', 'common.topic.image', 'm.03vbp89'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mis-Teeq'), ('Heartbreaker', 'music.composition.composer', 'Xavier Smith'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Avril Lavigne'), ('m.0pc67ly', 'award.award_nomination.award', 'MTV Movie Award for Best Jaw Dropping Moment'), ('Justin Bieber', 'common.topic.webpage', 'm.0r90dwg'), ('My Worlds Acoustic', 'music.album.releases', 'My Worlds Acoustic'), ('m.0zg8bc1', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Leif Garrett', 'people.person.profession', 'Singer'), ('Rihanna', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('My Worlds: The Collection', 'music.album.primary_release', 'My Worlds: The Collection'), ('Justin Bieber', 'music.artist.album', 'Never Say Never: The Remixes'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('radioIO Todays POP', 'common.topic.article', 'm.045zxs7'), ('Juicy J', 'music.composer.compositions', 'Lolly'), ('Alicia Keys', 'common.topic.notable_types', 'Musical Artist'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfpb_k'), ('All Bad', 'music.recording.song', 'All Bad'), ('Stuart Ford', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Duffy'), ('Lil Jon', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.episode', \"Judge's House No. 2\"), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Writer'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('LeAnn Rimes', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fvqf', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Lil Jon', 'broadcast.artist.content', 'HitzRadio.com'), ('Miley Cyrus', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3bl'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bnz6'), ('Victoria Justice', 'people.person.profession', 'Actor'), ('Teyana', 'freebase.valuenotation.has_value', 'Height'), ('Yves Bole', 'music.artist.album', 'Rain'), ('Heartbreaker', 'music.composition.composer', 'Chef Tone'), ('School Boy Records', 'common.topic.article', 'm.0gmd15q'), ('#thatPower', 'common.topic.notable_for', 'g.1ydntw0_k'), ('Ashley Tisdale', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Journals', 'music.album.primary_release', 'Journals'), ('Soulja Boy', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ray J'), ('m.0yrkc0l', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jay-Z'), ('1Club.FM: Power', 'broadcast.content.artist', 'Madonna'), ('Nicki Minaj', 'people.person.profession', 'Actor'), ('Madonna', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Jessica Simpson', 'music.artist.genre', 'Pop music'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhx1b'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Contemporary R&B', 'music.genre.parent_genre', 'Rhythm and blues'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Aaliyah'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me (album version)'), ('Linkin Park', 'broadcast.artist.content', '1.FM Top 40'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ricky Nelson', 'base.icons.icon.icon_genre', 'Teen idol'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Tommy Sands', 'common.topic.notable_types', 'Musical Artist'), ('Right Here', 'music.recording.artist', 'Justin Bieber'), ('m.0z8qx3w', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('m.0njhxd_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Avril Lavigne', 'broadcast.artist.content', 'Hot Wired Radio'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvpj'), ('David Cassidy', 'music.artist.genre', 'Pop music'), ('Yves Bole', 'music.artist.album', 'Moving On'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'JoJo'), ('m.0yrkr34', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Raekwon', 'people.person.profession', 'Actor'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Black Eyed Peas', 'music.artist.genre', 'Synthpop'), ('Disco', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gj1yzm'), ('Diplo', 'people.person.gender', 'Male'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0jvgmxc'), ('Runaway Love (remix)', 'music.recording.releases', 'Runaway Love (remix)'), ('Under the Mistletoe', 'music.album.genre', 'Rhythm and blues'), ('P!nk', 'people.person.profession', 'Musician'), ('m.0gbm3bs', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Maroon 5', 'music.artist.genre', 'Rock music'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'common.topic.notable_types', 'Musical Recording'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Roots'), ('CL', 'music.artist.genre', 'Hip hop music'), ('Selena Gomez', 'people.person.profession', 'Actor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Coldplay'), ('Change Me', 'music.composition.composer', 'Justin Bieber'), ('Confident', 'common.topic.notable_types', 'Musical Recording'), ('One Less Lonely Girl', 'music.album.compositions', 'One Less Lonely Girl'), ('m.0430821', 'common.webpage.category', 'Topic Webpage'), ('m.0njhvsq', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Shaun Cassidy', 'common.topic.notable_types', 'Musical Artist'), ('Eurodance', 'music.genre.parent_genre', 'Electronic dance music'), ('m.0yqfny6', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('Juvenile', 'people.person.gender', 'Male'), ('m.0njhvsq', 'award.award_honor.award_winner', 'Justin Bieber'), ('Barry Weiss', 'people.person.profession', 'Record producer'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_for', 'g.1yl5wcrp8'), ('Gwen Stefani', 'music.artist.genre', 'Synthpop'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Mary J. Blige', 'broadcast.artist.content', 'FLOW 103'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074plm'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('GotRadio - RnB Classics', 'broadcast.content.artist', 'Michael Jackson'), ('Shaffer Smith', 'broadcast.artist.content', '181-beat'), ('Michael Jackson', 'people.person.profession', 'Dancer'), (\"O'Kelly Isley, Jr.\", 'music.artist.genre', 'Rhythm and blues'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ray J', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Indie rock', 'common.topic.subject_of', 'Alan Motley'), ('Jessie J', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('P!nk', 'broadcast.artist.content', 'HitzRadio.com'), ('Amerie', 'people.person.profession', 'Singer'), ('Duffy', 'music.artist.genre', 'Rock music'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Hip hop music'), ('The Black Eyed Peas', 'broadcast.artist.content', 'WildFMRadio.com'), ('#thatPOWER', 'common.topic.notable_for', 'g.12pkcwrn5'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Wait For a Minute', 'music.recording.song', 'Wait for a Minute'), ('Hold Tight', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Urban contemporary', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Dance music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Hellogoodbye'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Club Mix)'), ('First Dance', 'music.composition.lyricist', 'Dwight Reynolds'), ('Bad 25', 'common.topic.notable_types', 'Film'), ('Thought Of You', 'music.recording.releases', 'Believe'), ('SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Rock music'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Singer', 'people.profession.specializations', 'Gospel singer'), ('Alicia Keys', 'music.artist.genre', 'Rhythm and blues'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All That Matters', 'common.topic.notable_types', 'Musical Album'), ('K-Ci & JoJo', 'people.person.gender', 'Male'), ('Nelly', 'broadcast.artist.content', '1Club.FM: Power'), ('Teen idol', 'common.topic.webpage', 'm.09wsj7g'), ('Ashley Tisdale', 'music.artist.genre', 'Teen pop'), ('Recovery', 'music.composition.composer', 'James Giannos'), ('Taylor Swift', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('Jordin Sparks', 'music.artist.genre', 'Teen pop'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Athan Grace'), ('Katy Perry', 'people.person.profession', 'Singer-songwriter'), ('m.0gxnnzy', 'celebrities.romantic_relationship.celebrity', 'Justin Bieber'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sean Paul'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0ghz3d6'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Miami', 'common.topic.notable_types', 'City/Town/Village'), ('m.0t4s_bn', 'award.award_honor.award_winner', 'Justin Bieber'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0_vw6nn', 'award.award_nomination.nominated_for', 'All That Matters'), ('Linkin Park', 'music.artist.genre', 'Electronic music'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Pray', 'music.composition.composer', 'Adam Messinger'), ('Hold Tight', 'music.composition.composer', 'Justin Bieber'), ('Victoria Justice', 'people.person.languages', 'English Language'), ('Stephen Melton', 'people.person.profession', 'Singer-songwriter'), ('Nelly Furtado', 'music.artist.genre', 'Rock music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Aqualung'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Big R Radio - Top 40 Hits', 'common.topic.notable_types', 'Broadcast Content'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Outkast'), ('Bryan-Michael Cox', 'people.person.profession', 'Singer'), ('Urban contemporary', 'broadcast.genre.content', '181-rnb'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0v90p2b'), ('London', 'location.location.time_zones', 'Eastern Time Zone'), ('m.0d33hsj', 'celebrities.friendship.friend', 'Christian Beadles'), ('Hot Wired Radio', 'broadcast.content.artist', 'Lady Gaga'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('This is Justin Bieber', 'film.film.language', 'English Language'), ('All Bad', 'common.topic.notable_types', 'Musical Album'), ('The Notorious B.I.G.', 'people.person.gender', 'Male'), ('Teen idol', 'base.icons.icon_genre.icons', 'Leif Garrett'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Jay-Z'), ('Madonna', 'music.artist.genre', 'Electronic dance music'), ('Gwen Stefani', 'music.artist.genre', 'Rock music'), ('Ciara', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('As Long as You Love Me', 'music.composition.composer', 'Nasri'), ('William Orbit', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lifehouse'), ('Verse Simmonds', 'music.composer.compositions', 'Confident'), ('Fabolous', 'freebase.valuenotation.has_value', 'Parents'), ('m.0zg2qjr', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('Savan Kotecha', 'freebase.valuenotation.has_value', 'Parents'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Mary J. Blige'), ('Baby', 'music.music_video.artist', 'Ludacris'), ('m.0ncpx3v', 'organization.leadership.organization', 'The Island Def Jam Music Group'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Flo Rida'), ('Eminem', 'broadcast.artist.content', 'Smoothbeats'), ('Drake', 'music.artist.genre', 'Hip hop music'), ('Nick Jonas', 'people.person.profession', 'Actor'), ('Miley Cyrus', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Eenie Meenie', 'music.composition.composer', 'Justin Bieber'), ('Eminem', 'people.person.profession', 'Record producer'), ('Playback Singer', 'people.profession.specialization_of', 'Singer'), ('Pattie Mallette', 'people.person.children', 'Justin Bieber'), ('The Black Eyed Peas', 'broadcast.artist.content', '.977 The Hits Channel'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Dub)'), ('Blackstreet', 'broadcast.artist.content', '.977 The Hits Channel'), ('PG (USA)', 'film.content_rating.country', 'United States of America'), ('Sean Kingston', 'people.person.profession', 'Musician'), ('L.A. Reid', 'people.person.nationality', 'United States of America'), ('m.0vp800w', 'music.recording_contribution.album', 'All Around the World'), ('Baby', 'music.album.contributor', 'm.0vmyv4w'), ('Little Bird', 'music.recording.featured_artists', 'Justin Bieber'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Akon'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.appearance_type', 'Host'), ('PYD', 'music.album.release_type', 'Single'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Freddy Fader'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.broadcast', 'Emphatic Radio.com! - 24kbps Stream'), ('Justin Bieber', 'celebrities.celebrity.sexual_relationships', 'm.0gxnnzy'), ('HitzRadio.com', 'broadcast.content.artist', 'Jurassic 5'), ('Chris Brown', 'music.artist.genre', 'Pop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fuel'), ('Ashlee Simpson', 'people.person.profession', 'Singer-songwriter'), ('The Isley Brothers', 'broadcast.artist.content', 'radioIO Classic RNB'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.artist', 'Justin Bieber'), ('As Long as You Love Me', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'Home to Mama'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jagged Edge'), ('Justin Timberlake', 'people.person.profession', 'Actor'), ('Justin Bieber', 'music.artist.album', 'Roller Coaster'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h5lzt'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jay-Z', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), (\"Bill O'Dowd\", 'film.producer.film', \"Justin Bieber's Believe\"), ('Leif Garrett', 'music.artist.genre', 'Disco'), ('m.0n4rmg7', 'award.award_honor.award_winner', 'Justin Bieber'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Dub)'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.artist.album', 'Boyfriend'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Dany Brillant'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d_'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gh0fdt'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Boyfriend', 'award.award_winning_work.awards_won', 'm.0yrkc0l'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Next to You', 'music.single.versions', 'Next to You'), ('m.0gxnp5d', 'base.popstra.hangout.business_location', 'Selena Gomez'), ('Jordan Pruitt', 'music.artist.genre', 'Rhythm and blues'), ('Bruce Dickinson', 'fictional_universe.fictional_character.gender', 'Male'), ('R. Kelly', 'music.artist.genre', 'Rhythm and blues'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('My World', 'music.album.release_type', 'EP'), ('Concert tour', 'freebase.type_profile.strict_included_types', 'Topic'), ('Michael Jackson', 'music.artist.genre', 'Disco'), ('Elvis Presley', 'people.person.gender', 'Male'), ('181-beat', 'broadcast.content.artist', 'Chris Brown'), ('m.0r90dwg', 'common.webpage.topic', 'Justin Bieber'), ('Benny Blanco', 'music.artist.genre', 'Synthpop'), ('Justin Bieber: Just Getting Started', 'book.book.editions', 'Justin Bieber: Just Getting Started'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Journals', 'music.album.release_type', 'Album'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Wont Stop (feat. Justin Bieber)', 'common.topic.notable_for', 'g.1s052gdth'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m57'), ('Actor', 'music.special_music_video_performance_type.special_music_video_performances', 'm.0z3vx9q'), ('m.0kt8f87', 'people.place_lived.person', 'Justin Bieber'), ('Pearl Jam', 'broadcast.artist.content', '1.FM Top 40'), ('Jeremy Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.01053qzf'), ('Coldplay', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Hip hop music', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.artist', 'Panic! at the Disco'), ('Sean Combs', 'common.topic.notable_types', 'Musical Artist'), ('Pray', 'common.topic.notable_for', 'g.12h2wqwc4'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Jay-Z'), ('Avery', 'people.person.profession', 'Singer-songwriter'), ('GotRadio - RnB Classics', 'broadcast.content.artist', 'The Isley Brothers'), ('1Club.FM: Channel One', 'common.topic.webpage', 'm.043082k'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Instrumental)'), ('m.0vmyv4w', 'music.recording_contribution.album', 'Baby'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('1Club.FM: V101', 'broadcast.content.artist', 'Jagged Edge'), ('Recovery', 'music.recording.artist', 'Justin Bieber'), ('Pray', 'common.topic.notable_types', 'Composition'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'Leona Lewis'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rob Thomas', 'music.artist.genre', 'Dance music'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Nelly'), ('Katy Perry', 'music.artist.genre', 'Europop'), ('Shaggy', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.012bm5cg', 'celebrities.rivalry.rival', 'Yves Bole'), ('R. Kelly', 'people.person.profession', 'Musician'), ('m.0gbm3cp', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('1Club.FM: Power', 'broadcast.content.artist', 'Twista'), ('J. Holiday', 'common.topic.notable_types', 'Musical Artist'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhvsq'), ('Scooter Braun', 'freebase.valuenotation.is_reviewed', 'Height'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Montell Jordan'), ('Thought Of You', 'music.composition.recordings', 'Thought Of You'), ('m.0gxnp72', 'base.popstra.support.supported_organization', 'Pattie Mallette'), ('Jessie J', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Drake', 'people.person.gender', 'Male'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juno Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0njvvj_'), ('Baby', 'music.album.genre', 'Eurodance'), ('Scooter Braun', 'film.person_or_entity_appearing_in_film.films', 'm.0101fsyr'), ('Kanye West', 'broadcast.artist.content', 'Hot 108 Jamz'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjhc6c'), ('Rob Thomas', 'common.topic.notable_types', 'Musical Artist'), ('The Pussycat Dolls', 'music.artist.genre', 'Contemporary R&B'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Beauty and a Beat', 'music.recording.releases', 'Believe'), ('Jason Mraz', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Vanessa Hudgens', 'people.person.gender', 'Female'), ('We Are the World 25 for Haiti', 'common.topic.notable_types', 'Musical Album'), ('Pattie Mallette', 'people.person.sibling_s', 'm.0wfn4pm'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('m.0n1ykxp', 'award.award_honor.honored_for', 'Baby'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108gy4p'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Dyce'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Dub Mix)'), ('m.0wjhc6c', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('.977 The Hits Channel', 'broadcast.content.genre', 'Pop music'), ('Ja Rule', 'broadcast.artist.content', 'Smoothbeats'), ('m.0njvvj_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k8y'), ('Justin Timberlake', 'music.artist.genre', 'Hip hop music'), ('Wait for a Minute', 'music.composition.composer', 'Tyga'), ('Image', 'type.property.schema', 'Topic'), ('m.0j8mhs_', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('m.0zbg2kw', 'award.award_nomination.ceremony', 'American Music Awards of 2010'), ('Synthpop', 'music.genre.parent_genre', 'Electronic dance music'), ('m.0z8qqh5', 'award.award_nomination.award', 'Teen Choice Award for Choice Music: Album Pop'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Never Say Never: The Remixes', 'common.topic.notable_for', 'g.125980jf9'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Boys Like Girls'), ('Jaxon Bieber', 'people.person.place_of_birth', 'Canada'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Ontario', 'base.biblioness.bibs_location.country', 'Canada'), ('Height', 'rdf-schema#range', 'Floating Point Number'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Hip hop music'), ('Dance music', 'broadcast.genre.content', 'C9 Radio'), ('Beauty and a Beat (Remixes)', 'common.topic.notable_for', 'g.126tkfc2x'), ('Carrie Underwood', 'people.person.nationality', 'United States of America'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Right Here', 'music.composition.recordings', 'Right Here'), ('Bigger', 'music.composition.lyricist', 'Dapo Torimiro'), ('m.0vp7mrq', 'music.recording_contribution.album', 'Runaway Love (remix)'), ('Justin Bieber', 'broadcast.artist.content', 'radioIO Todays POP'), ('Taylor Swift', 'music.artist.genre', 'Dance-pop'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0gbcs1_', 'tv.tv_guest_role.episodes_appeared_in', 'March 16, 2010'), ('Sean Kingston', 'people.person.profession', 'Record producer'), ('My Worlds Acoustic', 'music.album.genre', 'Pop music'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Justin Bieber'), ('Christmas in Washington', 'common.topic.notable_types', 'Film'), ('Justin Timberlake', 'people.person.profession', 'Record producer'), ('Right Here (featuring Drake)', 'music.recording.artist', 'Justin Bieber'), ('Jordin Sparks', 'people.person.profession', 'Model'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cajjmere Wray'), ('Jeremy Bieber', 'people.person.nationality', 'Canada'), ('HitzRadio.com', 'broadcast.content.artist', 'My Chemical Romance'), ('Janet Jackson', 'broadcast.artist.content', 'DeeGay'), ('C1', 'people.person.profession', 'Musician'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k7g'), ('Jaden Smith', 'music.featured_artist.recordings', 'Never Say Never'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_721l'), ('Rihanna', 'music.artist.genre', 'Urban contemporary'), ('m.0w5l5h1', 'education.education.student', 'Justin Bieber'), ('Justin Bieber', 'music.artist.album', 'Pray'), ('Avril Lavigne', 'people.person.profession', 'Model'), ('Adam Messinger', 'music.composer.compositions', 'Pray'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Bigger', 'music.recording.artist', 'Justin Bieber'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mariah Carey'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('Chef Tone', 'people.person.gender', 'Male'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Hoobastank'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Dan Cutforth'), ('Juelz Santana', 'music.artist.genre', 'Hip hop music'), ('Will i Am', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'music.featured_artist.albums', 'Lolly'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Fergie'), ('Jayceon Terrell Taylor', 'music.artist.genre', 'Contemporary R&B'), ('Beautiful', 'music.recording.artist', 'Justin Bieber'), ('One Time', 'music.album.releases', 'One Time'), ('m.0_sxby0', 'award.award_nomination.award', 'Shorty Award for Music'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jessie J', 'people.person.gender', 'Female'), ('m.0b497pt', 'common.webpage.topic', 'Record producer'), ('School Boy Records', 'music.record_label.artist', 'Justin Bieber'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Carrie Underwood', 'people.person.profession', 'Actor'), ('Jay-Z', 'people.person.profession', 'Film Producer'), ('Diplo', 'music.artist.genre', 'Electronic dance music'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Spanish Language', 'language.human_language.countries_spoken_in', 'United States of America'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Christina Aguilera', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0dm4cqr'), ('All Around The World (featuring Ludacris)', 'music.recording.featured_artists', 'Ludacris'), ('London', 'common.topic.notable_types', 'City/Town/Village'), ('Tupac Shakur', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'CKY'), ('HitzRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Gwen Stefani', 'broadcast.artist.content', '1Club.FM: Channel One'), ('P!nk', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Benny Blanco', 'music.artist.genre', 'Rhythm and blues'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Shaggy'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Tupac Shakur', 'freebase.valuenotation.has_no_value', 'Children'), ('Christina Milian', 'music.artist.genre', 'Dance music'), ('m.0jzrrqs', 'common.topic.notable_types', 'Location'), ('m.0tkqqgg', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Nelly', 'broadcast.artist.content', '181-beat'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw1tn'), ('Rihanna', 'music.artist.genre', 'Reggae'), ('Deborah Lurie', 'people.person.profession', 'Record producer'), ('Music Producer', 'common.topic.subjects', 'POPPMusic.net'), ('Record producer', 'common.topic.webpage', 'm.0b48fj5'), ('m.0njdq6k', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Nelly', 'people.person.profession', 'Singer'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Radio Mix)'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv7d'), ('Trey Songz', 'people.person.languages', 'English Language'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Lady Gaga', 'people.person.profession', 'Singer-songwriter'), ('m.0101fvq6', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Eenie Meenie'), ('Santana', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Jane Lipsitz', 'people.person.nationality', 'United States of America'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('#thatPOWER', 'music.album.compositions', '#thatPower'), ('Adam Messinger', 'people.person.profession', 'Musician'), ('Usher', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Sean Kingston', 'people.person.place_of_birth', 'Miami'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Contemporary R&B'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Profession'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Sean Paul'), ('Estelle', 'people.person.profession', 'Actor'), ('m.0pc670l', 'award.award_honor.award', 'MTV Movie Award for Best Jaw Dropping Moment'), ('Ernie Isley', 'freebase.valuenotation.has_value', 'Parents'), ('1.FM Top 40', 'broadcast.content.artist', 'Kylie Minogue'), ('My World 2.0', 'music.album.artist', 'Justin Bieber'), ('Beautiful', 'music.composition.recordings', 'Beautiful'), ('m.0101fv4c', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Ciara', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'music.artist.genre', 'Contemporary R&B'), ('American Music Award for Favorite Pop/Rock Male Artist', 'award.award_category.winners', 'm.0njdq6k'), ('Teen idol', 'base.icons.icon_genre.icons', 'Donny Osmond'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Teen idol', 'base.icons.icon_genre.icons', 'Shaun Cassidy'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Twista', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('PowerHitz', 'broadcast.content.artist', 'Leona Lewis'), ('Timbaland', 'people.person.profession', 'Record producer'), ('Mariah Carey', 'people.person.gender', 'Female'), ('Big R Radio - The Hawk', 'broadcast.content.artist', '50 Cent'), ('m.0kt8f87', 'people.place_lived.location', 'Stratford'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_706c'), ('Akon', 'broadcast.artist.content', 'HitzRadio.com'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Rubyhorse'), ('Jay Cassidy', 'people.person.gender', 'Male'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('CL', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'P!nk'), ('Pray', 'music.composition.recordings', 'Pray'), ('m.0101fs_z', 'film.personal_film_appearance.person', 'Jeremy Bieber'), ('Tupac Shakur', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Justin Timberlake'), ('Pray', 'music.composition.composer', 'Justin Bieber'), ('Vanessa Hudgens', 'music.artist.genre', 'Dance-pop'), ('Writer', 'common.topic.notable_types', 'Profession'), ('Red Hot Chili Peppers', 'music.artist.genre', 'Reggae'), ('Spouse', 'type.property.schema', 'Marriage'), ('Akon', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0hmvx8s', 'people.marriage.spouse', 'Jeremy Bieber'), ('Will i Am', 'music.artist.genre', 'Contemporary R&B'), ('Ronski Speed', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6zk4'), ('Justin Bieber', 'music.artist.album', 'Hold Tight'), ('Heartbreaker', 'common.topic.notable_for', 'g.12z651m29'), ('Live My Life', 'music.composition.recordings', 'Live My Life (Party Rock remix)'), ('Contemporary R&B', 'common.topic.image', 'Contemporary R&B singer Mary J. Blige performs on the National Mall during the “NFL Kickoff Live 2003” Concert'), ('Britney Spears', 'people.person.profession', 'Artist'), ('Lil Wayne', 'people.person.profession', 'Musician'), ('1Club.FM: V101', 'broadcast.content.artist', 'Whitney Houston'), ('Sean Kingston', 'music.artist.genre', 'Pop music'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Children'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Zendaya: Behind the Scenes', 'film.film.country', 'United States of America'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Musical Artist', 'freebase.documented_object.documentation', 'm.01z0mrz'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Avril Lavigne', 'people.person.profession', 'Musician'), ('JoJo', 'music.artist.genre', 'Hip hop music'), ('m.012bm4v7', 'celebrities.friendship.friend', 'Rita Ora'), ('Live My Life', 'music.album.releases', 'Live My Life'), ('Janet Jackson', 'people.person.profession', 'Record producer'), ('m.0y5t8gm', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Pop music', 'music.genre.parent_genre', 'Dance music'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('The Isley Brothers', 'music.artist.genre', 'Rhythm and blues'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Lady Gaga', 'broadcast.artist.content', 'radioIO Todays POP'), ('Lupe Fiasco', 'music.artist.origin', 'Chicago'), ('Ice Cube', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Power', 'broadcast.content.artist', 'Fergie'), ('m.0nhfd4m', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Singer-songwriter', 'people.profession.corresponding_type', 'Musical Artist'), ('PowerHitz', 'broadcast.content.artist', 'Lil Wayne'), ('Don Henley', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'common.topic.image', 'RadioIO.png'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0yqfny6'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_99fs'), ('The Roots', 'music.artist.genre', 'Hip hop music'), ('m.0gwhmhm', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Don Henley', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('As Long As You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Bad Day', 'music.album.releases', 'Bad Day'), ('Big R Radio - The Hawk', 'common.topic.image', 'Big R radio'), ('Jessie J', 'freebase.valuenotation.has_no_value', 'Children'), ('Janet Jackson', 'music.artist.label', 'Island Records'), ('m.0101ft2j', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('RedOne', 'music.artist.genre', 'Dance music'), ('Right Here', 'music.album.release_type', 'Single'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Gaga'), ('Miami Beach', 'common.topic.notable_types', 'City/Town/Village'), ('Never Say Never', 'common.topic.notable_types', 'Musical Recording'), ('Teen idol', 'base.icons.icon_genre.icons', 'Davy Jones'), ('m.0njw59_', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Bryan-Michael Cox', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('Bigger', 'music.composition.composer', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'P!nk'), ('radioIO Todays POP', 'broadcast.content.genre', 'Rock music'), ('Lady Gaga', 'freebase.valuenotation.has_no_value', 'Children'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Children'), ('Next to You', 'music.album.artist', 'Chris Brown'), ('Tricky Stewart', 'music.artist.genre', 'Hip hop music'), ('R. Kelly', 'people.person.nationality', 'United States of America'), ('Cory Gunz', 'music.artist.genre', 'Contemporary R&B'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('WildFMRadio.com', 'broadcast.content.artist', 'Rihanna'), ('m.0ng_k21', 'film.personal_film_appearance.film', 'Christmas in Washington'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaffer Smith', 'people.person.profession', 'Record producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jeff Klein'), ('Redfoo', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.genre', 'Rap music'), ('Beauty and a Beat', 'music.album.release_type', 'Single'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beautiful', 'common.topic.notable_for', 'g.126t625wd'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Instrumental)'), ('Willa Ford', 'people.person.nationality', 'United States of America'), ('Everlast', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'film.actor.film', 'm.0jztshx'), ('As Long as You Love Me', 'music.composition.language', 'English Language'), ('1.FM Top 40', 'broadcast.content.artist', 'Bryan Adams'), ('#Thatpower', 'music.recording.canonical_version', '#thatPOWER'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('My Worlds Acoustic', 'common.topic.notable_types', 'Musical Album'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0gbmnv1', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Leonardo DiCaprio', 'freebase.valuenotation.has_no_value', 'Children'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Place of birth'), (\"Bill O'Dowd\", 'people.person.profession', 'Film Producer'), ('Teen pop', 'music.genre.parent_genre', 'K-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0115qhzk'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty And A Beat', 'common.topic.notable_for', 'g.1258kxk_f'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shop Boyz'), ('Lil Jon', 'music.artist.genre', 'Hip hop music'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.artist', 'Gym Class Heroes'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_6_81'), ('m.0v1lwt2', 'tv.tv_guest_role.special_performance_type', 'Cameo appearance'), ('m.0y4tdml', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0gctwjk', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), ('Kelis', 'broadcast.artist.content', '1Club.FM: Power'), ('Montell Jordan', 'people.person.profession', 'Singer-songwriter'), ('m.0v30sv7', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'music.recording.tracks', 'Live My Life (Jaywalker remix)'), ('Baby', 'common.topic.article', 'm.09v3gb8'), ('Beauty and a Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Runaway Love (remix)', 'music.recording.artist', 'Kanye West'), ('Rihanna', 'music.artist.genre', 'Pop music'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordan Francis', 'music.artist.genre', 'Hip hop music'), ('Contemporary R&B', 'broadcast.genre.content', 'HitzRadio.com'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'common.topic.notable_types', 'Musical Recording'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Contemporary R&B'), ('m.0y84ynj', 'award.award_nomination.award_nominee', 'Will i Am'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Parents'), ('All Bad', 'common.topic.notable_types', 'Musical Recording'), ('Indie rock', 'common.topic.subjects', 'Singer-songwriter'), ('Dr. Dre', 'broadcast.artist.content', 'JellyRadio.com'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Snoop Dogg', 'people.person.profession', 'Film Producer'), (\"Destiny's Child\", 'music.artist.genre', 'Hip hop music'), ('Lolly', 'music.album.releases', 'Lolly'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Hot 108 Jamz'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Bad Boys Blue'), ('Musician', 'common.topic.subject_of', 'Artfarm'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ludacris'), ('Love Me', 'music.album.artist', 'Justin Bieber'), ('Adam Messinger', 'music.composer.compositions', 'Never Say Never'), ('Justin Bieber', 'common.topic.notable_for', 'g.1259ft7c9'), ('m.0z8t2dy', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('The Roots', 'common.topic.notable_types', 'Musical Artist'), ('Beauty and a Beat (Wideboys Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('1.FM Top 40', 'broadcast.content.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Rap music'), (\"Kids' Choice Award for Favorite Song\", 'award.award_category.winners', 'm.0sgk_cw'), ('Beauty And A Beat', 'music.composition.composer', 'Nicki Minaj'), ('1.FM Top 40', 'broadcast.content.artist', 'The Hoosiers'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0101fszs', 'film.personal_film_appearance.person', 'Pattie Mallette'), ('My World 2.0', 'music.album.genre', 'Contemporary R&B'), ('m.0yrhhqv', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrlqp1', 'freebase.valuenotation.has_value', 'Start Date'), ('Roller Coaster', 'music.recording.artist', 'Justin Bieber'), ('Shaffer Smith', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Pattie Mallette', 'common.topic.notable_for', 'g.125gnww48'), ('Profession', 'rdf-schema#domain', 'Person'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Janet Jackson'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Gender'), ('PYD', 'music.album.primary_release', 'PYD'), ('JoJo', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Top 40'), ('m.0njgyk4', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Location', 'type.type.properties', 'People born here'), ('The Isley Brothers', 'music.artist.genre', 'Disco'), ('Justin Bieber', 'common.topic.webpage', 'm.07lkzw7'), ('Clay Aiken', 'people.person.gender', 'Male'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Keyshia Cole', 'people.person.profession', 'Actor'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Roller Coaster', 'music.album.release_type', 'Single'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'Paramount Pictures'), ('m.0101ftm6', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Nelly', 'people.person.gender', 'Male'), ('Start Date', 'rdf-schema#range', 'Date/Time'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Blackstreet'), ('Britney Spears', 'people.person.profession', 'Singer-songwriter'), ('Die in Your Arms', 'music.album.release_type', 'Single'), ('Live My Life', 'music.composition.recorded_as_album', 'Live My Life'), ('Young Hollywood Award for Newcomer of the Year', 'award.award_category.winners', 'm.0yr9c1k'), ('Aaliyah', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Anastacia', 'music.artist.genre', 'Electronic dance music'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Trick Daddy', 'broadcast.artist.content', '1Club.FM: Power'), ('WildFMRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('Drake', 'people.person.nationality', 'Canada'), ('Rudolph Isley', 'music.composer.compositions', 'Bad Day'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Canada', 'location.country.first_level_divisions', 'Ontario'), ('Artist', 'people.profession.specializations', 'Dancer'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Mary J. Blige'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Amerie', 'music.artist.genre', 'Rhythm and blues'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Parents'), ('One Less Lonely Girl', 'music.album.genre', 'Rhythm and blues'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.composer.compositions', 'All That Matters'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.010lkp2z'), ('Nick Jonas', 'people.person.profession', 'Musician'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Teen Choice Award for Choice TV Villain', 'award.award_category.winners', 'm.0z898w6'), ('m.0yrjvlh', 'award.award_honor.award_winner', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'J. Holiday'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Ludacris'), ('m.0yrktlv', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Simple Plan'), ('Hot Wired Radio', 'broadcast.content.artist', 'Cascada'), ('Carrie Underwood', 'common.topic.notable_types', 'Musical Artist'), ('181-beat', 'broadcast.content.artist', 'Leona Lewis'), ('Deborah Lurie', 'freebase.valuenotation.has_value', 'Parents'), ('Cory Gunz', 'music.artist.genre', 'Rhythm and blues'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Official website'), (\"Bill O'Dowd\", 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World', 'music.album.genre', 'Synthpop'), ('MTV Europe Music Award for Best Push Artist', 'award.award_category.winners', 'm.0njw257'), ('Boyfriend', 'music.album.artist', 'Justin Bieber'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Aaliyah'), ('1Club.FM: Mix 106', 'broadcast.content.broadcast', '1Club.FM: Mix 106 - 64kbps Stream'), ('Benny Blanco', 'people.person.profession', 'Musician'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Justin Bieber'), ('David Cassidy', 'people.person.profession', 'Writer'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Savan Kotecha', 'music.artist.label', 'Island Records'), ('CL', 'people.person.profession', 'Dancer'), ('The Black Eyed Peas', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber', 'music.group_member.membership', 'm.011m26pv'), ('m.0njwqrb', 'award.award_honor.award', 'Queen Elizabeth II Diamond Jubilee Medal'), ('1Club.FM: Channel One', 'common.topic.notable_types', 'Broadcast Content'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Aaliyah'), ('Les Coulisses des Golden Globes', 'common.topic.notable_types', 'Film'), ('1Club.FM: Power', 'broadcast.content.artist', 'Nelly'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjgqck'), ('My Worlds Acoustic', 'common.topic.image', '220px-Myworldsacoustic.jpg'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Whitney Houston', 'people.person.profession', 'Actor'), ('Bigger', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Trey Songz', 'people.person.profession', 'Record producer'), ('Right Here', 'common.topic.notable_types', 'Canonical Version'), ('Chris Brown', 'people.person.profession', 'Singer'), ('One Less Lonely Girl', 'music.album.artist', 'Justin Bieber'), ('Fergie', 'music.artist.genre', 'Contemporary R&B'), ('Boyfriend', 'music.composition.recorded_as_album', 'Boyfriend'), ('Recovery', 'common.topic.notable_for', 'g.1yl5xlb58'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'people.person.places_lived', 'm.0gxnny8'), ('Spouse (or domestic partner)', 'type.property.expected_type', 'Marriage'), ('Official website', 'rdf-schema#range', 'URI'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_436'), ('Jessie J', 'music.artist.genre', 'Pop music'), ('m.064xzb1', 'common.webpage.category', 'Official Website'), ('Dr. Dre', 'people.person.nationality', 'United States of America'), ('Love Never Felt So Good', 'music.composition.form', 'Song'), ('Hip hop music', 'broadcast.genre.content', 'Hot Wired Radio'), ('m.0v90p2b', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_70rd'), ('Record producer', 'common.topic.subject_of', 'POPPMusic.net'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Rihanna'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Top 40'), ('2011 MTV Movie Awards', 'award.award_ceremony.awards_presented', 'm.0pc670l'), ('m.03hs4ny', 'common.webpage.topic', 'BeirutNights.com Radio'), ('Beyoncé Knowles', 'people.person.profession', 'Singer-songwriter'), ('m.0vb6hhj', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('iJustine', 'celebrities.celebrity.celebrity_rivals', 'm.012bm5cg'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'iJustine'), ('Usher', 'broadcast.artist.content', 'PowerHitz'), ('Next to You', 'music.album.album_content_type', 'Studio album'), ('All Around The World', 'common.topic.article', 'm.0j_3rq4'), ('Believe Acoustic', 'music.album.releases', 'Believe Acoustic'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Chris Brown', 'people.person.profession', 'Singer-songwriter'), ('Katy Perry: Part of Me', 'film.film.genre', 'Music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0h4yhbb'), ('Cris Cab', 'people.person.nationality', 'United States of America'), ('m.0ng_j6d', 'film.personal_film_appearance.film', 'This is Justin Bieber'), ('Aaliyah', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('1.FM Top 40', 'broadcast.content.artist', 'Kelly Clarkson'), ('Singer', 'people.profession.specializations', 'Dramatic coloratura'), ('Daniel Bedingfield', 'common.topic.notable_types', 'Musical Artist'), ('m.0ywvh8k', 'award.award_honor.honored_for', 'justinbieber'), ('Beautiful', 'music.recording.artist', 'Carly Rae Jepsen'), ('Enrique Iglesias', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Shaggy'), ('Chris Brown', 'music.artist.genre', 'Rhythm and blues'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Saturdays'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('Martin Kierszenbaum', 'common.topic.notable_types', 'Record Producer'), ('Jordin Sparks', 'people.person.languages', 'English Language'), ('DMX', 'people.person.nationality', 'United States of America'), ('Baby', 'music.composition.recordings', 'Baby'), ('Wait for a Minute', 'music.composition.composer', 'Justin Bieber'), ('Lady Gaga', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Confident', 'common.topic.notable_types', 'Musical Album'), ('My World', 'award.award_winning_work.awards_won', 'm.0z8s_wn'), ('Dance-pop', 'music.genre.parent_genre', 'K-pop'), ('Big Sean', 'music.featured_artist.albums', 'As Long as You Love Me'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber', 'broadcast.artist.content', '1.FM Top 40'), ('Justin Bieber', 'music.artist.album', 'Journals'), ('Justin Timberlake', 'broadcast.artist.content', 'radioIO Todays POP'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Usher', 'music.featured_artist.recordings', 'Somebody to Love'), ('Will Smith', 'broadcast.artist.content', '1Club.FM: Power'), ('Christian Beadles', 'people.person.nationality', 'United States of America'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Christina Milian', 'people.person.profession', 'Model'), ('Justin Timberlake', 'music.artist.genre', 'Pop music'), ('Boyfriend (Dada Life remix)', 'music.recording.artist', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Killers'), ('Pop music', 'music.genre.parent_genre', 'World music'), ('m.0njhxzc', 'award.award_honor.award_winner', 'Justin Bieber'), ('Lolly', 'common.topic.notable_types', 'Composition'), ('Colbie Caillat', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Goo Goo Dolls'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Parents'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('As Long As You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('Gas Pedal', 'music.recording.artist', 'Iamsu!'), ('1Club.FM: Power', 'broadcast.content.genre', 'Pop music'), ('Sir Mix-a-Lot', 'common.topic.notable_types', 'Musical Artist'), ('Mistletoe', 'music.album.releases', 'Mistletoe'), ('Stephen Melton', 'common.topic.subjects', 'Pop music'), ('Janet Jackson', 'broadcast.artist.content', '1Club.FM: V101'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Height'), ('Hold Tight', 'common.topic.notable_types', 'Musical Album'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Children', 'type.property.expected_type', 'Person'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Thought Of You', 'common.topic.notable_types', 'Composition'), ('Chingy', 'broadcast.artist.content', 'radioIO Todays RNB'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'FLOW 103'), ('First Dance', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Gwen Stefani'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'P!nk'), ('2011 MTV Video Music Aid Japan', 'award.award_ceremony.awards_presented', 'm.0yrhrwc'), ('Donna Summer', 'people.person.profession', 'Singer'), ('Green Day', 'broadcast.artist.content', 'HitzRadio.com'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0430821', 'common.webpage.topic', '1Club.FM: Mix 106'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yqfny6'), ('m.0gbm3cx', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njdq6k'), ('Height', 'type.property.expected_type', 'Floating Point Number'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Brandon Blue Hamilton'), ('Pray', 'music.album.release_type', 'Single'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Gas Pedal', 'music.recording.featured_artists', 'Iamsu!'), ('FLOW 103', 'broadcast.content.artist', 'Snoop Dogg'), ('Blu Cantrell', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.featured_artist.recordings', 'g.1q6708pcc'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhxd_'), ('Lady Antebellum', 'broadcast.artist.content', '.977 The Hits Channel'), ('Will Smith', 'people.person.languages', 'English Language'), ('Snoop Dogg', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'people.person.profession', 'Film Producer'), ('Jaden Smith', 'music.featured_artist.recordings', 'Never Say Never (acoustic)'), ('Teyana', 'music.artist.genre', 'Hip hop music'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Stu Fisher'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'music.album.featured_artists', 'Ludacris'), ('Beauty and a Beat (Remixes)', 'music.album.artist', 'Justin Bieber'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.episode', 'Semi-Final'), ('Boyfriend', 'common.topic.notable_types', 'Composition'), ('m.0njhxd_', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhx1b'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Beyoncé Knowles'), ('The Isley Brothers', 'music.artist.label', 'The Island Def Jam Music Group'), ('Nelly', 'broadcast.artist.content', 'HitzRadio.com'), ('Hit-Boy', 'freebase.valuenotation.has_value', 'Parents'), ('Soulja Boy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Height'), ('Dany Brillant', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('JellyRadio.com', 'broadcast.content.artist', '112'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Wait For a Minute', 'music.album.releases', 'Wait For a Minute'), ('All Around the World', 'music.album.releases', 'All Around the World'), ('Singer', 'base.schemastaging.context_name.pronunciation', 'g.125_qgwzx'), ('m.012nv5gz', 'people.place_lived.person', 'Yves Bole'), ('Katy Perry', 'music.artist.genre', 'Rock music'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Brandy Norwood', 'broadcast.artist.content', '1Club.FM: V101'), ('Pattie Mallette', 'people.person.parents', 'Michael Mallette'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Pray', 'common.topic.notable_types', 'Musical Album'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Nelly Furtado'), ('United States Dollar', 'base.coinsdaily.coin_type.country', 'United States of America'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010nwpx1'), ('Composition', 'type.type.properties', 'Lyricist'), ('Yves Bole', 'influence.influence_node.influenced_by', 'Jessie J'), ('Enrique Iglesias', 'music.artist.origin', 'Miami'), ('justinbieber', 'internet.blog.blogger', 'Justin Bieber'), ('m.0pc670l', 'award.award_honor.honored_for', 'Justin Bieber: Never Say Never'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jrb'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot remix)'), ('Never Say Never', 'music.single.versions', 'Never Say Never (acoustic)'), ('Big Sean', 'people.person.gender', 'Male'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Official website'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Height'), ('m.0gxnnwp', 'people.sibling_relationship.sibling', 'Jaxon Bieber'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rap music'), ('U Smile', 'common.topic.notable_types', 'Musical Album'), ('Singer-songwriter', 'common.topic.subject_of', 'Indie rock'), ('1Club.FM: 80s (Pop)', 'common.topic.notable_types', 'Broadcast Content'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'BWO'), ('m.0z8s_wn', 'award.award_honor.ceremony', 'NME Awards 2011'), ('Beauty and a Beat', 'music.album.album_content_type', 'Studio album'), ('Amerie', 'music.artist.genre', 'Contemporary R&B'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Leona Lewis'), ('Ashley Tisdale', 'music.artist.genre', 'Pop music'), ('m.0nfnx_3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('m.0tk76mf', 'award.award_nomination.ceremony', 'Juno Awards of 2011'), ('Rita Ora', 'celebrities.celebrity.celebrity_friends', 'm.012bm3j9'), ('m.0z8s562', 'award.award_honor.award', 'NME Award for Worst Dressed'), ('Jayceon Terrell Taylor', 'people.person.gender', 'Male'), ('Mariah Carey', 'broadcast.artist.content', 'radioIO RNB Mix'), ('m.0z83x9l', 'award.award_honor.award_winner', 'Justin Bieber'), ('All Bad', 'music.album.primary_release', 'All Bad'), ('Khalil', 'common.topic.notable_types', 'Musical Artist'), ('Recovery', 'music.composition.composer', 'Justin Bieber'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Britney Spears', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Hip hop music', 'broadcast.genre.content', 'HitzRadio.com'), ('K-Ci & JoJo', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Justin Bieber: Never Say Never', 'film.film.rating', 'G (USA)'), ('Sean Kingston', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Fabian', 'common.topic.notable_types', 'Musical Artist'), (\"Dick Clark's Primetime New Year's Rockin' Eve 2013\", 'common.topic.notable_types', 'Film'), ('Hot Wired Radio', 'broadcast.content.artist', 'Fergie'), ('m.0z1ndbl', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Beautiful', 'music.composition.lyricist', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjvlh'), ('My Worlds Acoustic', 'music.album.genre', 'Contemporary R&B'), ('Actor', 'common.topic.notable_types', 'Profession'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjk4'), ('HitzRadio.com', 'broadcast.content.artist', 'Gwen Stefani'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat', 'music.album.artist', 'Justin Bieber'), ('Heartbreaker', 'music.composition.composer', 'Maejor'), ('Little Bird', 'music.recording.canonical_version', 'Live My Life'), ('radioIO Todays RNB', 'broadcast.content.producer', 'Radioio'), ('Jason Mraz', 'broadcast.artist.content', 'HitzRadio.com'), (\"Destiny's Child\", 'broadcast.artist.content', '.977 The Hits Channel'), ('My World', 'music.album.genre', 'Rhythm and blues'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('m.0f0dwc4', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('The Pussycat Dolls', 'music.artist.genre', 'Dance-pop'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'music.artist.genre', 'Hip hop music'), ('Runaway Love (remix)', 'music.recording.song', 'Runaway Love'), ('Singer', 'people.profession.part_of_professional_field', 'Singing'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Believe', 'common.topic.article', 'm.0j263y7'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'P!nk'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('50 Cent', 'broadcast.artist.content', 'HitzRadio.com'), ('Leonardo DiCaprio', 'people.person.languages', 'English Language'), ('m.0yrhrwc', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrhhqv', 'award.award_honor.ceremony', '2010 MTV Video Music Brazil'), ('Never Let You Go', 'music.recording.artist', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'August Rigo'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Kylie Minogue', 'people.person.gender', 'Female'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Carrie Underwood', 'broadcast.artist.content', 'Hot Wired Radio'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Anna Nalick'), ('Musician', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('Justin Timberlake', 'music.artist.genre', 'Teen pop'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mike Oldfield'), ('2013 Radio Disney Music Awards', 'award.award_ceremony.awards_presented', 'm.0y4tdml'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jane Lipsitz', 'film.director.film', 'Katy Perry: Part of Me'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'common.topic.notable_types', 'Musical Recording'), ('Pray', 'music.composition.recordings', 'Pray'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Micko Larkin'), ('Diplo', 'people.person.profession', 'Record producer'), ('1.FM Top 40', 'broadcast.content.artist', 'Lighthouse Family'), ('Believe Tour', 'common.topic.article', 'm.0j_3p6y'), ('m.0tk76mf', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Heartbreaker', 'music.album.releases', 'Heartbreaker'), ('Justin Bieber', 'music.featured_artist.recordings', 'g.11btzxy8qx'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Jeremy Bieber', 'people.person.spouse_s', 'm.0hmvx8s'), ('m.03zb5cw', 'common.webpage.category', 'Topic Webpage'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Power'), ('Katy Perry', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0z87d3n', 'award.award_honor.award_winner', 'Justin Bieber'), ('Montell Jordan', 'people.person.profession', 'Actor'), ('JellyRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('Justin Bieber', 'music.artist.album', 'My World 2.0'), ('Sean Kingston', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.film_production_design_by', 'Tom E. Marzullo'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Club Mix)'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0y6dqtf'), ('Benny Blanco', 'people.person.nationality', 'United States of America'), ('Live My Life', 'common.topic.notable_for', 'g.126sk4331'), ('Chris Brown', 'music.artist.album', 'Next to You'), ('Miley Cyrus', 'music.artist.genre', 'Synthpop'), ('My Worlds Acoustic', 'common.topic.notable_for', 'g.125h3jvrd'), ('Roller Coaster', 'music.composition.composer', 'Josh Gudwin'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Donna Summer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Buckcherry'), ('m.0gbm3fj', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Usher'), ('Sheryl Crow', 'music.artist.genre', 'Pop music'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Lolly', 'music.single.versions', 'Lolly'), ('1Club.FM: V101', 'broadcast.content.artist', 'Mariah Carey'), ('Actor', 'common.topic.subjects', 'Bleona'), ('As Long as You Love Me', 'music.composition.composer', 'Andre Lindal'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('50 Cent', 'music.artist.genre', 'Hip hop music'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njw1tn', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Ja Rule', 'broadcast.artist.content', 'HitzRadio.com'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fergie'), ('Juicy J', 'music.artist.genre', 'Hip hop music'), ('Chris Brown', 'broadcast.artist.content', '181-beat'), ('Runaway Love (remix)', 'music.album.releases', 'Runaway Love (remix)'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Gender'), ('CL', 'people.person.gender', 'Female'), ('Runaway Love (remix)', 'music.album.release_type', 'Single'), ('Usher', 'broadcast.artist.content', '1Club.FM: Power'), ('Disco', 'common.topic.notable_types', 'Musical genre'), ('Demi Lovato', 'music.artist.genre', 'Dance music'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Smoothbeats'), ('Juno Awards of 2011', 'award.award_ceremony.awards_presented', 'm.0njvtth'), ('Clay Aiken', 'people.person.profession', 'Actor'), ('Usher', 'music.featured_artist.recordings', 'First Dance'), ('Elvis Presley', 'people.person.profession', 'Singer'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ciara', 'broadcast.artist.content', 'HitzRadio.com'), ('Juvenile', 'freebase.valuenotation.has_value', 'Children'), ('Yves Bole', 'people.person.sibling_s', 'm.0129jzth'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('m.09wwfnk', 'common.webpage.topic', 'Record producer'), ('Justin Bieber', 'book.author.works_written', 'Justin Bieber: First Step 2 Forever'), ('1.FM Top 40', 'broadcast.content.genre', 'Rock music'), ('World music', 'common.topic.notable_types', 'Musical genre'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.09wnj4l', 'common.webpage.topic', 'Teen idol'), ('Fergie', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Enrique Iglesias', 'music.artist.genre', 'Dance music'), ('Next to You', 'common.topic.notable_types', 'Musical Recording'), ('Martin Kierszenbaum', 'music.composer.compositions', 'Live My Life'), ('Will i Am', 'broadcast.artist.content', 'Hot Wired Radio'), ('My World 2.0', 'music.album.release_type', 'Album'), ('WildFMRadio.com', 'broadcast.content.artist', 'Ciara'), ('Record producer', 'common.topic.webpage', 'm.09xwqmn'), ('Pras', 'people.person.profession', 'Actor'), ('Chef Tone', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'L.A. Reid'), ('Dan Cutforth', 'people.person.profession', 'Film Producer'), ('Ontario', 'base.aareas.schema.administrative_area.administrative_parent', 'Canada'), ('Runaway Love (remix)', 'music.album.contributor', 'm.0vp7mrq'), ('m.05sp3_n', 'business.company_name_change.company', 'The Island Def Jam Music Group'), ('Shaffer Smith', 'people.person.profession', 'Dancer'), ('All That Matters (video)', 'music.recording.artist', 'Justin Bieber'), ('Alanis Morissette', 'people.person.profession', 'Singer-songwriter'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Gloria Gaynor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Ja Rule', 'common.topic.notable_types', 'Musical Artist'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Beyoncé Knowles'), ('Sir Mix-a-Lot', 'people.person.profession', 'Record producer'), ('Justin Timberlake', 'music.artist.genre', 'Contemporary R&B'), ('Kelly Clarkson', 'music.artist.genre', 'Dance-pop'), ('Rihanna', 'music.artist.genre', 'Electronic dance music'), ('As Long as You Love Me (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jennifer Lopez', 'people.person.languages', 'Spanish Language'), ('Beautiful', 'common.topic.notable_types', 'Composition'), ('Willa Ford', 'people.person.profession', 'Actor'), ('Usher', 'people.person.profession', 'Record producer'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fsz2'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0v1lwt2'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Taylor Swift', 'people.person.profession', 'Actor'), ('Believe', 'music.album.artist', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'Paramount Pictures'), ('Heartbreaker', 'music.recording.song', 'Heartbreaker'), ('m.0njgyk4', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Album'), ('Teyana', 'people.person.gender', 'Female'), ('Chris Jasper', 'freebase.valuenotation.has_value', 'Parents'), ('Foreign Remix', 'common.topic.notable_for', 'g.1q67phzjh'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miley Cyrus', 'people.person.nationality', 'United States of America'), ('Shaggy', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Johnny Crawford', 'people.person.profession', 'Musician'), ('m.0njhxd_', 'award.award_honor.award', 'Billboard Music Award for Top Digital Media Artist'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.artist', 'Justin Bieber'), ('Rihanna', 'music.artist.genre', 'Rhythm and blues'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cherish'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Confident', 'music.recording.featured_artists', 'Chance the Rapper'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Date of birth'), (\"1Club.FM: Jammin' Oldies\", 'common.topic.notable_types', 'Broadcast Content'), ('PYD', 'common.topic.notable_types', 'Musical Album'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0w3gbtv'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011j_d79'), ('English Language', 'language.human_language.main_country', 'Canada'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('Lady Antebellum', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Official website', 'rdf-schema#domain', 'Topic'), ('School Boy Records', 'music.record_label.artist', 'Carly Rae Jepsen'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat'), ('m.0ywtfj6', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0sgkw_d', 'freebase.valuenotation.has_no_value', 'Winning work'), ('1Club.FM: Mix 106', 'common.topic.webpage', 'm.0430821'), ('Justin Bieber', 'music.artist.album', 'All That Matters'), ('Estelle', 'music.artist.genre', 'Electronic dance music'), ('m.0z85qxq', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.appearance_type', 'Him/Herself'), ('Ronald Isley', 'music.composer.compositions', 'Bad Day'), ('Jaden Smith', 'people.person.gender', 'Male'), ('m.0dj34q5', 'common.webpage.category', 'About'), ('Selena Gomez', 'music.artist.genre', 'Pop music'), ('PowerHitz', 'common.topic.notable_types', 'Broadcast Content'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc0sf'), ('Gwen Stefani', 'people.person.nationality', 'United States of America'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Buckcherry'), ('All Bad', 'common.topic.notable_types', 'Composition'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Never Say Never', 'common.topic.article', 'm.0c3vvnp'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Hold Tight', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Jason Mraz', 'music.artist.genre', 'Rock music'), ('JellyRadio.com', 'broadcast.content.artist', 'Jayceon Terrell Taylor'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien Luvstep mix)'), ('Kelis', 'music.artist.genre', 'Dance-pop'), ('Nelly Furtado', 'music.artist.genre', 'Pop music'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Award category'), ('m.09xwqmn', 'common.webpage.topic', 'Record producer'), ('Rudolph Isley', 'people.person.nationality', 'United States of America'), ('Eenie Meenie', 'common.topic.notable_types', 'Canonical Version'), ('Die in Your Arms', 'music.composition.composer', 'Kelly Lumpkins'), ('Singing', 'people.professional_field.professions_in_this_field', 'Singer'), ('Nelly Furtado', 'music.artist.genre', 'Contemporary R&B'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Estelle', 'music.artist.genre', 'Dance-pop'), ('Under the Mistletoe', 'common.topic.notable_for', 'g.1255l1vtl'), ('m.0gbmnsc', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Sir Mix-a-Lot'), ('All Bad', 'music.composition.composer', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvzb'), ('Nelly Furtado', 'people.person.nationality', 'Canada'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Pop music'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Young Hollywood Award for Newcomer of the Year', 'award.award_category.nominees', 'm.0yr9tjb'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0pbzq13', 'film.performance.character', 'Alien on TV Monitors #2'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Rise to Fame', 'common.topic.notable_types', 'Film'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njdq6k'), ('Under the Mistletoe', 'music.album.genre', 'Pop music'), ('Brandy Norwood', 'people.person.gender', 'Female'), ('Kid Cudi', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Rihanna', 'broadcast.artist.content', 'Hot Wired Radio'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ciara'), ('Jane Lipsitz', 'people.person.profession', 'Film Producer'), ('Chingy', 'broadcast.artist.content', '.977 The Hits Channel'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('American Music Awards of 2012', 'award.award_ceremony.nominees', 'm.0n58kgb'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Matchbox Twenty'), ('Ontario', 'location.location.time_zones', 'Eastern Time Zone'), ('One Less Lonely Girl', 'music.album.releases', 'One Less Lonely Girl'), ('Ludacris', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Abriel'), ('Keyshia Cole', 'people.person.languages', 'English Language'), ('Dany Brillant', 'people.person.profession', 'Singer'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.composer.compositions', 'As Long as You Love Me'), ('Jordan Francis', 'music.artist.genre', 'Teen pop'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Daniel Bedingfield', 'broadcast.artist.content', '1Club.FM: Power'), ('Jessica Simpson', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'music.artist.album', 'My Worlds'), ('Keyshia Cole', 'people.person.gender', 'Female'), ('Toby Gad', 'people.person.gender', 'Male'), ('Where Are Ü Now', 'common.topic.notable_for', 'g.11b7fxl7j1'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Cascada'), ('Lupe Fiasco', 'freebase.valuenotation.has_value', 'Parents'), ('Live My Life (Jaywalker remix)', 'music.recording.artist', 'Far East Movement'), ('RBMG Records', 'music.record_label.artist', 'Scooter Braun'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0yrk18w', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Ciara', 'music.artist.genre', 'Contemporary R&B'), ('PYD', 'music.composition.composer', 'Justin Bieber'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Power'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Musician', 'common.topic.subject_of', 'Stephen Melton'), ('Beauty and a Beat (Remixes)', 'music.album.album_content_type', 'Remix album'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.composition.recordings', 'Baby'), ('radioIO RNB Mix', 'broadcast.content.location', 'Tampa'), ('Lolly', 'music.album.release_type', 'Single'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0z87d3n'), ('m.0gwhmhm', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0v_714c', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Winning work', 'type.property.expected_type', 'Award-Winning Work'), ('Lil Jon', 'people.person.nationality', 'United States of America'), ('All Bad', 'common.topic.notable_for', 'g.1yl5r3f4x'), ('Area codes 519 and 226', 'location.location.containedby', 'Ontario'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjgrzc'), ('Lady Gaga', 'broadcast.artist.content', 'PowerHitz'), ('Alicia Keys', 'music.artist.genre', 'Pop music'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j2189_'), ('Baby', 'music.composition.recordings', 'Baby'), ('m.0yrkc0l', 'award.award_honor.honored_for', 'Boyfriend'), ('Britney Spears', 'broadcast.artist.content', 'radioIO Todays POP'), ('Jason Mraz', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yr9c1k'), ('Ginuwine', 'broadcast.artist.content', '1Club.FM: V101'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Amerie', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z83x9l'), ('justinbieber', 'common.topic.notable_types', 'Award-Winning Work'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0tjyljn'), ('JoJo', 'people.person.profession', 'Singer-songwriter'), ('JoJo', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('CL', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.01053qzf'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Author', 'freebase.type_hints.included_types', 'Person'), ('Bad Day', 'music.album.primary_release', 'Bad Day'), ('#thatPOWER', 'music.album.artist', 'Will i Am'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: V101'), ('Johnny Crawford', 'people.person.gender', 'Male'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Hip hop music', 'broadcast.genre.content', 'JellyRadio.com'), ('m.0y5th3r', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'music.artist.genre', 'Dance music'), ('Lil Wayne', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long As You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Carrie Underwood', 'people.person.profession', 'Singer-songwriter'), ('Mason Levy', 'freebase.valuenotation.has_value', 'Date of birth'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jet Black Stare'), ('Tyga', 'music.composer.compositions', 'Wait for a Minute'), ('m.0j8z6tl', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Mariah Carey', 'people.person.profession', 'Musician'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien dubstep edit)'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien dubstep edit)'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('Justin bieber', 'common.image.appears_in_topic_gallery', 'Justin Bieber (VEVO Channel)'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Height'), ('Enrique Iglesias', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhxzc'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'MTV Films'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8755b'), ('Gas Pedal', 'common.topic.notable_for', 'g.11b5l_z7r1'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv74'), ('Never Say Never (acoustic)', 'music.recording.artist', 'Justin Bieber'), ('Never Say Never: The Remixes', 'common.topic.article', 'm.0g9y40j'), ('m.0njw4z2', 'freebase.valuenotation.has_no_value', 'Winning work'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Chingy'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Boyfriend', 'award.award_nominated_work.award_nominations', 'm.0v1d2xz'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Parents', 'type.property.expected_type', 'Person'), ('Justin Bieber: Never Say Never', 'film.film.music', 'Edvard Grieg'), ('Iggy Azalea', 'freebase.valuenotation.has_value', 'Parents'), ('justinbieber', 'common.topic.notable_types', 'Blog'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fsyr'), ('Whitney Houston', 'broadcast.artist.content', '1Club.FM: V101'), ('Michael Jackson', 'music.artist.genre', 'Contemporary R&B'), ('Nicki Minaj', 'music.artist.contribution', 'm.0vp755b'), ('Elvis Presley', 'people.person.nationality', 'United States of America'), ('Baby', 'music.single.versions', 'Baby'), ('Under the Mistletoe', 'music.album.album_content_type', 'Studio album'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jon McLaughlin'), ('m.0yrkgd6', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Documentary film'), ('The Roots', 'music.artist.label', 'The Island Def Jam Music Group'), ('Michael Jackson', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cobra Starship'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beast'), ('Jordin Sparks', 'people.person.profession', 'Singer-songwriter'), ('Next to You', 'music.recording.artist', 'Chris Brown'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Selena Gomez', 'freebase.valuenotation.has_no_value', 'Children'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.song', 'As Long as You Love Me'), ('Justin Timberlake', 'broadcast.artist.content', 'FLOW 103'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Ciara'), ('Baby', 'common.topic.notable_types', 'Award-Winning Work'), ('American Music Award for Artist of the Year', 'award.award_category.winners', 'm.0njdns_'), ('Adam Messinger', 'music.lyricist.lyrics_written', 'That Should Be Me'), ('Rita Ora', 'people.person.profession', 'Actor'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Johnny Crawford', 'people.person.languages', 'English Language'), ('One Time', 'common.topic.notable_for', 'g.1256drntj'), ('m.0gbmnsk', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('2010 MTV Video Music Brazil', 'award.award_ceremony.awards_presented', 'm.0yrhhqv'), ('Madonna', 'people.person.profession', 'Film Producer'), ('Carrie Underwood', 'people.person.gender', 'Female'), ('Sean Combs', 'music.artist.genre', 'Hip hop music'), ('m.0101fszb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Kings of Leon', 'music.artist.genre', 'Indie rock'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Justin Bieber'), ('m.0z87597', 'award.award_nomination.ceremony', 'NME Awards 2012'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Selena Gomez'), ('Year', 'rdf-schema#range', 'Date/Time'), ('Demi Lovato', 'people.person.profession', 'Musician'), ('Ernie Isley', 'people.person.profession', 'Singer'), ('Trick Daddy', 'people.person.profession', 'Record producer'), ('Dance music', 'broadcast.genre.content', 'NonStopPlay.com'), ('Enrique Iglesias', 'music.artist.genre', 'Electronic dance music'), ('Britney Spears', 'people.person.profession', 'Record producer'), ('Bigger', 'music.composition.form', 'Song'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', \"Destiny's Child\"), ('Jessie J', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sean Paul'), ('Khalil', 'music.artist.genre', 'Rhythm and blues'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Chris Brown'), ('Lolly', 'common.topic.notable_for', 'g.11b75p5t_1'), ('Terius Nash', 'music.composer.compositions', 'Baby'), ('Gas Pedal', 'music.recording.featured_artists', 'Justin Bieber'), ('John Mamann', 'people.person.profession', 'Musician'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_4d6'), ('Yves Bole', 'music.artist.album', 'Sunday Morning'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'KooL CrAzE'), ('Die in Your Arms', 'music.recording.artist', 'Justin Bieber'), ('30 Days in May', 'film.film.personal_appearances', 'm.0y5tj13'), ('Justin Bieber', 'music.composer.compositions', 'Recovery'), ('PYD', 'music.recording.artist', 'Justin Bieber'), ('One Time', 'music.album.primary_release', 'One Time'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Lady Antebellum'), ('m.09y89l2', 'common.webpage.topic', 'Teen idol'), ('Miley Cyrus', 'base.icons.icon.icon_genre', 'Teen idol'), ('Kelly Clarkson', 'broadcast.artist.content', 'radioIO Todays POP'), ('Under the Mistletoe', 'music.album.genre', 'Christmas music'), ('.977 The Hits Channel', 'broadcast.content.artist', 'No Doubt'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Lil Jon', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (Dada Life remix)'), ('m.0vp7cl4', 'music.recording_contribution.contributor', 'Usher'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Hikaru Utada', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'music.artist.album', 'Change Me'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvhv'), ('Kanye West', 'music.artist.origin', 'Chicago'), ('m.0y4tdml', 'award.award_honor.award', 'Radio Disney Music Award for Best Male Artist'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0ndc259', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Album'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Ronald Isley', 'music.artist.label', 'The Island Def Jam Music Group'), ('Cory Gunz', 'music.artist.genre', 'Pop music'), ('50 Cent', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: 80s (Pop)', 'broadcast.content.location', 'Chicago'), ('Gwen Stefani', 'music.artist.genre', 'Hip hop music'), ('Ray J', 'people.person.profession', 'Singer'), ('Jon M. Chu', 'people.person.profession', 'Film Producer'), ('Rihanna', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'music.artist.genre', 'Contemporary R&B'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Juvenile'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Hoobastank'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Country of nationality'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Boys Like Girls'), ('1.FM Top 40', 'broadcast.content.artist', 'Don Henley'), ('Nicki Minaj', 'people.person.profession', 'Artist'), ('Trey Songz', 'music.artist.genre', 'Hip hop music'), ('Katy Perry: Part of Me', 'film.film.language', 'English Language'), ('181-beat', 'broadcast.content.artist', 'Fabolous'), ('Fabolous', 'broadcast.artist.content', 'HitzRadio.com'), ('Believe Acoustic', 'award.award_nominated_work.award_nominations', 'm.0v30sv7'), ('Demi Lovato', 'music.artist.genre', 'Dance-pop'), ('Anastacia', 'people.person.place_of_birth', 'Chicago'), ('#thatPower', 'award.award_nominated_work.award_nominations', 'm.0y84ynj'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Anastacia'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3g2'), ('Profession', 'owl#inverseOf', 'People With This Profession'), ('Tricky Stewart', 'people.person.profession', 'Record producer'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0jztshx', 'film.performance.special_performance_type', 'Him/Herself'), ('Teen idol', 'base.icons.icon_genre.icons', 'Christina Aguilera'), ('Rob Thomas', 'music.artist.origin', 'United States of America'), ('Love Me', 'common.topic.notable_types', 'Musical Album'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('First Dance', 'music.composition.composer', 'Justin Bieber'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Lady Gaga'), ('m.0sgk_cw', 'award.award_honor.honored_for', 'Baby'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.artist.album', 'Bad Day'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvk9'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (acoustic version)'), ('Miley Cyrus', 'celebrities.celebrity.celebrity_friends', 'm.0dm4cqr'), ('#thatPower', 'music.recording.tracks', '#thatPower'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_70fx'), ('Person', 'type.type.properties', 'Country of nationality'), ('m.0pc670l', 'award.award_honor.ceremony', '2011 MTV Movie Awards'), ('Next to You', 'music.recording.artist', 'Chris Brown'), ('m.0gbm3g2', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Author', 'freebase.type_profile.strict_included_types', 'Topic'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Contemporary R&B'), ('Raekwon', 'people.person.profession', 'Musician'), ('Chris Brown', 'broadcast.artist.content', 'Sunshine Radio'), ('Right Here (featuring Drake)', 'common.topic.notable_types', 'Musical Recording'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.composer', 'Justin Bieber'), ('Gavin DeGraw', 'people.person.profession', 'Artist'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0wjgqck', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Akon', 'people.person.profession', 'Record producer'), ('Journals', 'freebase.valuenotation.is_reviewed', 'Artist'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Big Pun'), ('Katy Perry', 'people.person.languages', 'English Language'), ('Contemporary R&B', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Smoothbeats', 'broadcast.content.genre', 'Urban contemporary'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Vanessa Hudgens'), ('Jay Cassidy', 'people.person.place_of_birth', 'Chicago'), ('m.0gfmm45', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0sgkw8v', 'freebase.valuenotation.has_no_value', 'Winning work'), ('HitzRadio.com', 'broadcast.content.genre', 'Top 40'), ('m.0y5tl39', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Aaliyah'), ('Selena Gomez', 'music.artist.genre', 'Synthpop'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnv1'), ('Justin Bieber', 'people.person.gender', 'Male'), ('Eenie Meenie', 'music.album.release_type', 'Single'), ('Yves Bole', 'people.person.profession', 'Blogger'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg2w8r'), ('Rhythm and blues', 'music.genre.subgenre', 'Contemporary R&B'), ('R. Kelly', 'people.person.languages', 'English Language'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Trey Songz', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fv74', 'film.film_regional_release_date.film_release_region', 'Canada'), ('Christina Aguilera', 'people.person.profession', 'Singer'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('m.0ndc0sf', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Funkmaster Flex'), ('Timbaland', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.album', 'Mistletoe'), ('Chance the Rapper', 'music.artist.origin', 'Chicago'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98t5'), ('Ginuwine', 'music.artist.genre', 'Contemporary R&B'), ('m.0yrjvlh', 'award.award_honor.award', 'Teen Choice Award for Choice Music: Breakout Artist - Male'), ('Elvis Presley', 'base.icons.icon.icon_genre', 'Teen idol'), ('Yves Bole', 'people.person.parents', 'Marjan Raseni'), ('Live My Life (Party Rock remix)', 'common.topic.notable_for', 'g.1yg9d9djx'), ('The Black Eyed Peas', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7m'), ('Dr. Dre', 'broadcast.artist.content', 'HitzRadio.com'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft1d'), ('City/Town/Village', 'freebase.type_hints.included_types', 'Location'), ('CL', 'music.artist.genre', 'K-pop'), ('Never Say Never', 'music.recording.song', 'Never Say Never'), ('m.0njdns_', 'award.award_honor.award', 'American Music Award for Artist of the Year'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teen idol', 'base.icons.icon_genre.icons', 'Zac Efron'), ('HitzRadio.com', 'broadcast.content.artist', 'DHT'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Fergie', 'music.artist.genre', 'Rhythm and blues'), ('Trey Songz', 'people.person.profession', 'Actor'), ('m.0y803nt', 'award.award_honor.award', 'Brit Award for International Breakthrough Act'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Jay-Z', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Rodney Jerkins', 'people.person.profession', 'Musician'), ('Adrienne Bailon', 'music.artist.label', 'The Island Def Jam Music Group'), ('Shaggy', 'people.person.profession', 'Musician'), ('Roller Coaster', 'music.recording.song', 'Roller Coaster'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3f5'), ('Justin Bieber', 'common.topic.webpage', 'm.0bvmhvb'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.episode', 'Season Finale Part 2'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Yves Bole', 'base.mediaextended.youtube_channel.subscribers', 'm.0123p2z1'), ('#thatPower', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'All Bad'), ('1Club.FM: Power', 'broadcast.content.artist', 'Timbaland'), ('Dance music', 'broadcast.genre.content', 'BeirutNights.com Radio'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Fall Out Boy'), ('Yves Bole', 'influence.influence_node.influenced', 'Whitney Houston'), ('m.0yr9c1k', 'award.award_honor.ceremony', '2010 Young Hollywood Awards'), ('Beyoncé Knowles', 'people.person.profession', 'Singer'), ('Kylie Minogue', 'music.artist.genre', 'Contemporary R&B'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Hot Wired Radio', 'broadcast.content.artist', 'Kelly Clarkson'), ('Keyshia Cole', 'people.person.profession', 'Musician'), ('Scottish Canadian', 'common.topic.notable_types', 'Ethnicity'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Paramore'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bryan-Michael Cox', 'music.composer.compositions', 'Never Let You Go'), ('Smoothbeats', 'broadcast.content.genre', 'Hip hop music'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0sxhgqj'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jwx'), ('WildFMRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0sgk_cw', 'award.award_honor.award_winner', 'Justin Bieber'), ('1Club.FM: Mix 106', 'broadcast.content.location', 'Chicago'), ('Khalil', 'music.artist.genre', 'Pop music'), ('m.0v90p2b', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0pcr2dh', 'tv.tv_guest_role.character', 'Jason McCann'), ('Weblink', 'type.property.authorities', 'm.0x_'), ('Justin Bieber', 'music.artist.album', 'Love Never Felt So Good'), ('Hip hop music', 'broadcast.genre.content', '1.FM Top 40'), ('Emphatic Radio.com!', 'common.topic.notable_types', 'Broadcast Content'), ('Fergie', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Juelz Santana', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber', 'music.artist.contribution', 'm.0vpggkk'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('That Should Be Me', 'common.topic.notable_types', 'Composition'), ('Red Hot Chili Peppers', 'broadcast.artist.content', 'radioIO Todays POP'), ('Nick Jonas', 'people.person.languages', 'English Language'), ('Live My Life', 'music.composition.form', 'Song'), ('David Nicksay', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Writer', 'people.profession.corresponding_type', 'Author'), ('As Long As You Love Me', 'music.recording.artist', 'Big Sean'), ('Turn to You (Mother’s Day Dedication)', 'music.album.releases', 'Turn to You (Mother’s Day Dedication)'), ('Live My Life (Party Rock remix)', 'music.recording.canonical_version', 'Live My Life'), ('Hikaru Utada', 'music.artist.genre', 'Hip hop music'), ('m.0wjhc6c', 'award.award_honor.award_winner', 'Justin Bieber'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Akon', 'music.artist.genre', 'Hip hop music'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Height'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber: Never Say Never', 'common.topic.article', 'm.0g9tcbl'), ('Taylor Swift', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Sean Kingston', 'music.artist.genre', 'Reggae'), ('Right Here', 'music.album.compositions', 'Right Here'), ('Scooter Braun', 'people.person.profession', 'Film Producer'), ('Dapo Torimiro', 'music.artist.genre', 'Dance music'), (\"Justin Bieber's Believe\", 'common.topic.notable_types', 'Film'), ('Runaway Love (remix)', 'music.album.artist', 'Kanye West'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Daniel Bedingfield'), ('radioIO RNB Mix', 'common.topic.image', 'RadioIO.png'), ('m.010htdc2', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Juelz Santana'), ('First Dance', 'music.composition.lyricist', 'Usher'), ('m.0z0tmyv', 'award.award_honor.award_winner', 'Justin Bieber'), ('FLOW 103', 'common.topic.notable_types', 'Broadcast Content'), ('Sean Combs', 'broadcast.artist.content', '1Club.FM: Power'), ('Ja Rule', 'music.artist.genre', 'Hip hop music'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_72js'), ('Frank Sinatra', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Hikaru Utada'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Keyshia Cole'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Bruce Dickinson'), ('m.0cvc8k4', 'common.webpage.topic', 'My World 2.0'), ('m.0_srv2b', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('WildFMRadio.com', 'broadcast.content.artist', 'Usher'), ('Teen idol', 'common.topic.webpage', 'm.09wnj4l'), ('As Long As You Love Me', 'music.recording.artist', 'Justin Bieber'), ('My Worlds: The Collection', 'common.topic.article', 'm.0gh8t69'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('Emphatic Radio.com!', 'common.topic.article', 'm.03hs0f2'), ('Justin Bieber', 'base.popstra.celebrity.breakup', 'm.0gxnp26'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Album'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Sarah Landman'), ('As Long As You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('1Club.FM: Channel One', 'broadcast.content.broadcast', '1Club.FM: Channel One - 64kbps Stream'), ('The Black Eyed Peas', 'broadcast.artist.content', 'HitzRadio.com'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Date of birth'), ('Chance the Rapper', 'people.person.gender', 'Male'), ('m.0129j_53', 'education.education.student', 'Yves Bole'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Lady Gaga'), ('Kylie Minogue', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkw_d'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Coldplay', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Stephen Melton', 'music.artist.genre', 'Rock music'), ('Willa Ford', 'people.person.profession', 'Singer'), ('As Long as You Love Me', 'common.topic.notable_for', 'g.1257jn46q'), ('NME Award for Worst Album', 'award.award_category.nominees', 'm.0z87597'), ('m.0z85qxq', 'award.award_nomination.nominated_for', 'Die in Your Arms'), ('Cris Cab', 'people.person.place_of_birth', 'Miami'), ('Frank Sinatra', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Rise to Fame', 'film.film.genre', 'Documentary film'), ('#thatPOWER', 'music.recording.tracks', '#thatpower'), ('Rock music', 'common.topic.subject_of', 'Stephen Melton'), ('Akon', 'music.artist.genre', 'Rhythm and blues'), ('Canada', 'location.country.languages_spoken', 'Spanish Language'), ('justinbieber', 'internet.blog.blogger', 'Justin Bieber'), ('Gwen Stefani', 'broadcast.artist.content', 'radioIO Todays POP'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Demi Lovato', 'people.person.profession', 'Singer-songwriter'), ('Ice Cube', 'people.person.profession', 'Film Producer'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Year'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award category'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0z8t2dy'), (\"2012 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkw8v'), ('Die in Your Arms', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'One Time'), ('Rodney Jerkins', 'people.person.profession', 'Record producer'), ('My World 2.0', 'common.topic.notable_types', 'Musical Album'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Ja Rule'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.genre', 'Music'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Matchbox Twenty'), ('Justin Bieber', 'music.featured_artist.recordings', 'Next to You'), ('m.0v90skf', 'award.award_honor.award_winner', 'Justin Bieber'), ('L.A. Reid', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('New Kids on the Block', 'music.artist.genre', 'Contemporary R&B'), ('Cris Cab', 'music.artist.label', 'Island Records'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Chloe Bridges', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Sheryl Crow', 'people.person.profession', 'Record producer'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.0z0tgz6', 'award.award_nomination.ceremony', '4th Annual Shorty Awards'), ('Britney Spears', 'people.person.gender', 'Female'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'OneRepublic'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Lolly', 'common.topic.notable_for', 'g.1yfp1cfhq'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Date of birth'), (\"O'Kelly Isley, Jr.\", 'people.person.profession', 'Singer-songwriter'), ('Runaway Love (remix)', 'music.recording.artist', 'Justin Bieber'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrktlv'), ('Rihanna', 'broadcast.artist.content', 'HitzRadio.com'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber: First Step 2 Forever', 'book.written_work.original_language', 'English Language'), ('WildFMRadio.com', 'broadcast.content.artist', 'Flo Rida'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Blackstreet', 'common.topic.notable_types', 'Musical Artist'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Will Smith'), ('m.0yrhmll', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('Deborah Lurie', 'film.music_contributor.film', 'Katy Perry: Part of Me'), ('Sheryl Crow', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0d33hsj', 'celebrities.friendship.friend', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jennifer Lopez'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Beautiful'), ('Singer', 'common.topic.subject_of', 'Alan Motley'), ('Favorite Girl', 'music.album.release_type', 'Single'), ('Stephen Melton', 'people.person.languages', 'English Language'), ('New Kids on the Block', 'music.artist.genre', 'Teen pop'), ('m.0z8t2dy', 'award.award_nomination.ceremony', 'NME Awards 2011'), ('Drake', 'people.person.profession', 'Record producer'), ('Believe', 'music.album.genre', 'Contemporary R&B'), ('m.0101fszs', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Colbie Caillat', 'people.person.languages', 'English Language'), ('m.07lkzw7', 'common.webpage.resource', 'm.0bldgxn'), ('Jayceon Terrell Taylor', 'common.topic.notable_types', 'Musical Artist'), ('m.0njdq6k', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Male Artist'), ('Justin Bieber', 'people.person.sibling_s', 'm.0gxnnwc'), ('Nelly', 'broadcast.artist.content', 'PowerHitz'), ('Beauty and a Beast', 'music.recording.canonical_version', 'Beauty and a Beat'), ('m.0gbm3d3', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Baby', 'music.music_video.music_video_song', 'Baby'), ('Michael Jackson', 'music.artist.genre', 'Electronic music'), ('Red Hot Chili Peppers', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Christina Aguilera', 'people.person.gender', 'Female'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'The Roots'), ('Jordan Pruitt', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('radioIO RNB Mix', 'broadcast.content.producer', 'Radioio'), ('My Worlds Acoustic', 'music.album.primary_release', 'My Worlds Acoustic'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Drake', 'music.artist.genre', 'Contemporary R&B'), ('Baby', 'music.composition.form', 'Song'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 32kbps Stream'), ('m.0101ft2j', 'film.personal_film_appearance.person', 'Rodney Jerkins'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkgd6'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rihanna'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5tl39'), ('Marvin Isley', 'people.person.profession', 'Musician'), ('Beauty and a Beat (Bisbetic Remix)', 'common.topic.notable_types', 'Musical Recording'), ('m.0yrktlv', 'award.award_honor.award_winner', 'Justin Bieber'), ('Chris Brown', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Taylor Swift', 'people.person.gender', 'Female'), ('Whitney Houston', 'influence.influence_node.influenced_by', 'Madonna'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Official website'), ('August Rigo', 'people.person.gender', 'Male'), ('Brandy Norwood', 'people.person.profession', 'Singer-songwriter'), ('Michael Jackson', 'influence.influence_node.influenced', 'Lady Gaga'), ('Right Here', 'music.album.artist', 'Justin Bieber'), ('Pearl Jam', 'music.artist.genre', 'Rock music'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Bryan-Michael Cox', 'music.producer.releases_produced', 'My World'), ('Ludacris', 'music.featured_artist.albums', 'Baby'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Terius Nash', 'music.artist.genre', 'Pop music'), ('PYD', 'common.topic.notable_for', 'g.1yp3ccjx9'), ('Chance the Rapper', 'music.featured_artist.recordings', 'Confident'), ('Nelly', 'music.artist.genre', 'Contemporary R&B'), ('HitzRadio.com', 'broadcast.content.artist', 'Puddle of Mudd'), ('Wont Stop (feat. Justin Bieber)', 'common.topic.notable_types', 'Musical Recording'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The All-American Rejects'), ('Don Henley', 'freebase.valuenotation.has_value', 'Parents'), ('Mariah Carey', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Alice Deejay'), ('Gwen Stefani', 'music.artist.genre', 'Dance-pop'), ('Ashlee Simpson', 'music.artist.genre', 'Electronic music'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('Mariah Carey', 'people.person.profession', 'Actor'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Jordan Francis'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Musical Album', 'type.type.properties', 'Album content type'), ('Max Martin', 'music.artist.genre', 'Dance-pop'), ('August Rigo', 'people.person.profession', 'Singer'), ('radioIO Todays RNB', 'common.topic.notable_types', 'Broadcast Content'), ('m.064xzb1', 'common.webpage.topic', 'The Island Def Jam Music Group'), ('Justin Bieber', 'people.person.profession', 'Actor'), ('Avery', 'people.person.gender', 'Female'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beast'), ('Urban contemporary', 'broadcast.genre.content', 'JellyRadio.com'), ('Don Henley', 'music.artist.genre', 'Rock music'), ('m.0njw1tn', 'award.award_honor.ceremony', '2010 MTV Europe Music Awards'), ('Juicy J', 'people.person.profession', 'Record producer'), ('Victoria Justice', 'music.artist.genre', 'Pop music'), ('Justin Bieber Concerts', 'common.resource.annotations', 'm.0r90dwg'), ('Maroon 5', 'common.topic.notable_types', 'Musical Artist'), ('Estelle', 'people.person.profession', 'Singer-songwriter'), ('m.0z898w6', 'award.award_honor.award', 'Teen Choice Award for Choice TV Villain'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv15'), ('m.0j21b47', 'music.music_video_performance.music_video_character', 'Singer'), ('m.0101ft5f', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('m.0vp7mrq', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Jay-Z', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('Contemporary R&B', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('Bad Day', 'music.composition.composer', 'Rudolph Isley'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Music & Concert Movies'), ('m.0_w3zrs', 'award.award_nomination.nominated_for', 'Hold Tight'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Rap music'), ('m.0njw59_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Where Are Ü Now', 'music.recording.featured_artists', 'Diplo'), ('Date of birth', 'type.property.schema', 'Person'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Mariah Carey'), ('Justin Bieber: Never Say Never', 'film.film.cinematography', 'Reed Smoot'), ('Janet Jackson', 'people.person.gender', 'Female'), ('All Bad', 'music.album.releases', 'All Bad'), ('Electronic music', 'common.topic.notable_types', 'Musical genre'), ('Turn to You (Mother’s Day Dedication)', 'common.topic.notable_for', 'g.12blrwnvw'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('Nathan Lanier', 'film.music_contributor.film', \"Justin Bieber's Believe\"), ('Max Martin', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft1d'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d9'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Shaft'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Jennifer Lopez', 'people.person.profession', 'Dancer'), ('The Isley Brothers', 'music.artist.genre', 'Contemporary R&B'), ('Nasri', 'music.composer.compositions', 'As Long as You Love Me'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Club Mix)'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('Raekwon', 'music.artist.genre', 'Hip hop music'), ('Michael Jackson', 'people.person.profession', 'Musician'), ('William Orbit', 'music.artist.genre', 'Trance music'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0t4syfh'), ('m.0yrkc0l', 'award.award_honor.award', 'Teen Choice Award for Choice Single: Male Artist'), ('Maroon 5', 'music.artist.genre', 'Indie rock'), ('Justin Bieber', 'base.popstra.celebrity.supporter', 'm.0gxnp72'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Teyana', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'M.I.A.'), ('As Long as You Love Me (album version)', 'music.recording.song', 'As Long as You Love Me'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Ronald Isley', 'people.person.nationality', 'United States of America'), ('m.0pbzq13', 'film.performance.film', 'Men in Black 3'), ('Smoothbeats', 'broadcast.content.genre', 'Rap music'), ('m.0sgkrj4', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Bigger', 'music.recording.song', 'Bigger'), ('2011 MTV Movie Awards', 'award.award_ceremony.nominees', 'm.0pc67ly'), ('Chris Jasper', 'people.person.profession', 'Record producer'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njhxzc'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: 80s (Pop)'), ('Aaliyah', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Change Me', 'music.composition.composer', 'Andre Harris'), ('Everlast', 'people.person.profession', 'Musician'), ('Alanis Morissette', 'broadcast.artist.content', 'radioIO Todays POP'), ('Kelly Clarkson', 'broadcast.artist.content', '1.FM Top 40'), ('Believe Acoustic', 'music.album.primary_release', 'Believe Acoustic'), ('Thought Of You', 'music.recording.canonical_version', 'Thought of You'), ('Paul Anka', 'people.person.gender', 'Male'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Hot Wired Radio'), ('Trick Daddy', 'people.person.gender', 'Male'), ('m.0yrk0mt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Ciara'), ('Katy Perry', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0y80n25', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Alanis Morissette', 'people.person.nationality', 'Canada'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw4z2'), ('Hot Wired Radio', 'broadcast.content.artist', 'Leona Lewis'), ('m.0sgkyfg', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('As Long As You Love Me (Audiobot remix)', 'common.topic.notable_types', 'Musical Recording'), ('Alanis Morissette', 'people.person.profession', 'Singer'), ('Geri Halliwell', 'common.topic.notable_types', 'Musical Artist'), ('Artist', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('Right Here', 'music.composition.recorded_as_album', 'Right Here'), ('Book', 'freebase.type_hints.included_types', 'Written Work'), ('Rodney Jerkins', 'common.topic.notable_types', 'Record Producer'), ('Justin Bieber: First Step 2 Forever', 'book.written_work.author', 'Justin Bieber'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Believe', 'music.album.releases', 'Believe'), ('Jaden Smith', 'freebase.valuenotation.has_no_value', 'Children'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njhxzc', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Daniel Bedingfield', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Paul Anka', 'music.artist.genre', 'Rock music'), ('Willa Ford', 'music.artist.genre', 'Dance-pop'), ('Fergie', 'music.artist.genre', 'Dance-pop'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Chris Brown'), ('1Club.FM: V101', 'broadcast.content.artist', 'Ginuwine'), ('Jaden Smith', 'people.person.parents', 'Will Smith'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_types', 'Canonical Version'), ('PYD', 'music.recording.featured_artists', 'R. Kelly'), ('m.0101ftt1', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('2010 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw257'), ('Never Let You Go', 'music.composition.composer', 'Justin Bieber'), ('Smoothbeats', 'broadcast.content.genre', 'Hip hop music'), ('Singer-songwriter', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Robby Stewart'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('1Club.FM: Power', 'broadcast.content.artist', 'Chris Brown'), ('August Rigo', 'music.artist.genre', 'Contemporary R&B'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rudolph Valentino', 'people.person.nationality', 'Italy'), ('#thatPOWER', 'music.recording.tracks', 'That Power'), ('Next to You', 'common.topic.notable_types', 'Musical Album'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_vw2_d'), ('m.0y5t8gm', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kelis'), ('Hot Wired Radio', 'broadcast.content.artist', 'Hinder'), ('m.012bm5cg', 'celebrities.rivalry.rival', 'iJustine'), ('.977 The Hits Channel', 'common.topic.notable_types', 'Broadcast Content'), ('Place of birth', 'type.property.reverse_property', 'People born here'), ('Justin Bieber: Just Getting Started', 'common.topic.article', 'm.0n1s4c2'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('#thatPower', 'music.recording.artist', 'Will i Am'), ('Stratford', 'location.location.time_zones', 'Eastern Time Zone'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Black Cards'), ('J. Holiday', 'people.person.profession', 'Singer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Leona Lewis'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Blackstreet'), ('m.0101ft1r', 'film.personal_film_appearance.person', 'Will i Am'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8qx3w'), ('Justin Bieber', 'music.artist.album', 'My World'), ('Gwen Stefani', 'people.person.profession', 'Dancer'), ('Baby', 'music.composition.composer', 'Ludacris'), ('Yves Bole', 'people.person.employment_history', 'm.012nv3hv'), ('Canada', 'location.country.official_language', 'English Language'), ('Live My Life', 'music.album.primary_release', 'Live My Life'), ('Rita Ora', 'music.artist.genre', 'Rock music'), ('Mason Levy', 'people.person.profession', 'Record producer'), ('Gwen Stefani', 'broadcast.artist.content', 'HitzRadio.com'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.recordings', \"Turn to You (Mother's Day Dedication)\"), ('Hot Wired Radio', 'broadcast.content.artist', 'Britney Spears'), ('Fergie', 'broadcast.artist.content', 'Hot Wired Radio'), ('Contemporary R&B', 'broadcast.genre.content', 'FLOW 103'), ('Scooter Braun', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3fr'), ('August Rigo', 'music.artist.genre', 'Dance-pop'), ('Donna Summer', 'people.person.profession', 'Actor'), ('Aaliyah', 'music.artist.genre', 'Hip hop music'), ('Urban contemporary', 'common.topic.notable_types', 'Musical genre'), ('Rita Ora', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0101ft1d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvbf'), ('Montell Jordan', 'people.person.profession', 'Singer'), ('m.0sgkw_d', 'award.award_honor.ceremony', \"2012 Kids' Choice Awards\"), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Groove Coverage'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me (acoustic version)'), ('Fall Out Boy', 'music.artist.genre', 'Pop music'), ('m.0v30sv7', 'award.award_nomination.nominated_for', 'Believe Acoustic'), ('Donna Summer', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Wait for a Minute', 'music.composition.recordings', 'Wait For a Minute'), ('m.0njhtjj', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Timberlake', 'people.person.profession', 'Dancer'), ('m.0njvtth', 'award.award_honor.ceremony', 'Juno Awards of 2011'), ('Roller Coaster', 'music.composition.composer', 'Rodney Jerkins'), ('Shorty Award for Celebrity', 'award.award_category.winners', 'm.0y_g556'), ('Lolly', 'music.album.featured_artists', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Rock music'), ('m.0101ftk6', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teyana', 'people.person.profession', 'Dancer'), ('Gwen Stefani', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Pattie Mallette', 'people.person.gender', 'Female'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('First Dance', 'music.composition.lyricist', 'Alexander Parhm, Jr.'), ('Donna Summer', 'people.person.profession', 'Singer-songwriter'), ('Alien on TV Monitors #2', 'film.film_character.portrayed_in_films', 'm.0pbzq13'), ('m.0yrhhqv', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Urban contemporary', 'broadcast.genre.content', 'PowerHitz'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3fj'), ('m.0gbcs1_', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0wjgrzc', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Judy Garland', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'music.composer.compositions', 'Lolly'), ('#Thatpower', 'music.recording.canonical_version', '#thatPOWER'), ('Ice Cube', 'music.artist.genre', 'Hip hop music'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0ywvh8k', 'award.award_honor.award', 'Shorty Award for Celebrity'), ('Kid Cudi', 'people.person.nationality', 'United States of America'), ('Usher', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3b7'), ('m.0gbm3dn', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Contemporary R&B', 'broadcast.genre.content', 'GotRadio - RnB Classics'), ('m.0yr9c1k', 'award.award_honor.award', 'Young Hollywood Award for Newcomer of the Year'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Mary J. Blige'), ('Nelly Furtado', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0sxhq3d', 'award.award_nomination.nominated_for', 'Believe'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0ywvh8k'), ('Alicia Keys', 'people.person.nationality', 'United States of America'), ('My World', 'common.topic.notable_types', 'Musical Album'), ('Mannie Fresh', 'music.artist.genre', 'Hip hop music'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Chris Brown', 'broadcast.artist.content', '.977 The Hits Channel'), ('All Bad', 'music.album.artist', 'Justin Bieber'), ('Bryan-Michael Cox', 'people.person.place_of_birth', 'Miami'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordin Sparks', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Change Me', 'music.recording.song', 'Change Me'), ('m.0dj34q5', 'common.webpage.topic', 'Justin Bieber'), ('Boyfriend', 'music.album.compositions', 'Boyfriend'), ('50 Cent', 'people.person.profession', 'Actor'), ('Christina Milian', 'broadcast.artist.content', '1.FM Top 40'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftrz'), ('Wait for a Minute', 'common.topic.notable_for', 'g.1yl5t5h0r'), ('Christina Aguilera', 'broadcast.artist.content', '1.FM Top 40'), ('Jessica Simpson', 'broadcast.artist.content', 'radioIO Todays POP'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Shaggy'), ('Will i Am', 'music.artist.genre', 'Hip hop music'), ('John Mamann', 'people.person.profession', 'Singer'), ('Frank Sinatra', 'people.person.profession', 'Film Producer'), ('DMX', 'music.artist.genre', 'Hip hop music'), ('Mariah Carey', 'music.artist.genre', 'Hip hop music'), ('Lil Wayne', 'broadcast.artist.content', 'PowerHitz'), ('Justin Bieber', 'music.composer.compositions', 'Roller Coaster'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Justin Timberlake'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Avril Lavigne'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_syttc'), ('Johntá Austin', 'music.artist.label', 'Island Records'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Gavin DeGraw'), ('Never Let You Go', 'common.topic.notable_for', 'g.1z2stmrpq'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mark Morrison'), ('#thatPOWER', 'music.album.featured_artists', 'Justin Bieber'), ('m.0z87d3n', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvtth'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Profession'), (\"O'Kelly Isley, Jr.\", 'music.artist.genre', 'Rock music'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkrj4'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Pop music'), ('Alanis Morissette', 'people.person.profession', 'Musician'), ('BeirutNights.com Radio', 'common.topic.notable_for', 'g.1259zkvd0'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk4gn'), ('All Around the World', 'common.topic.notable_for', 'g.12v_hb5zc'), ('m.0101ft1d', 'film.personal_film_appearance.person', 'Max Martin'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjsq'), ('Baby', 'common.topic.notable_for', 'g.125g47b9s'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk18w'), ('First Dance', 'music.composition.lyricist', 'Jesse Wilson'), ('2013 Radio Disney Music Awards', 'award.award_ceremony.nominees', 'm.0_vmmj6'), ('Sean Kingston', 'music.artist.genre', 'Rhythm and blues'), ('Kuk Harrell', 'people.person.place_of_birth', 'Chicago'), ('m.0yrkgd6', 'award.award_honor.award_winner', 'Justin Bieber'), ('First Dance', 'freebase.valuenotation.is_reviewed', 'Lyricist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sara Bareilles'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Jay-Z'), ('m.0n1ykxp', 'award.award_honor.award_winner', 'Justin Bieber'), ('Keyshia Cole', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft3d'), ('Yves Bole', 'music.artist.album', 'Look Inside Of Me'), ('JellyRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('One Time', 'music.album.releases', 'One Time'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Trance music'), ('Singer', 'common.topic.image', \"Ercole de' Roberti 003\"), ('Chicago', 'base.biblioness.bibs_topic.subsumes', 'Chicago'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Gender'), ('CL', 'music.artist.label', 'School Boy Records'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('Billboard Music Award for Top Male Artist', 'award.award_category.winners', 'm.0v90skf'), ('Rihanna', 'music.artist.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.artist', 'Shaffer Smith'), ('Redfoo', 'music.featured_artist.recordings', 'Live My Life (Party Rock remix)'), ('Boyfriend', 'music.album.releases', 'Boyfriend'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Nicki Minaj', 'music.artist.genre', 'Contemporary R&B'), ('Hold Tight', 'common.topic.notable_for', 'g.1yg4dn910'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'Douglas C. Merrifield'), ('Turn to You (Mother’s Day Dedication)', 'music.album.releases', 'Turn to You (Mother’s Day Dedication)'), ('#Thatpower', 'music.recording.featured_artists', 'Justin Bieber'), ('Timbaland', 'broadcast.artist.content', 'WildFMRadio.com'), ('R. Kelly', 'people.person.profession', 'Singer-songwriter'), ('m.012m1vnr', 'music.group_membership.role', 'Record producer'), ('m.0dm4cqr', 'celebrities.friendship.friend', 'Justin Bieber'), ('Baby', 'music.album.primary_release', 'Baby'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw4z2'), ('Donna Summer', 'music.artist.genre', 'Dance music'), ('m.0ndc259', 'award.award_honor.honored_for', 'Believe'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Height'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Everlast', 'people.person.gender', 'Male'), ('Fabolous', 'music.artist.genre', 'Hip hop music'), ('Under the Mistletoe', 'music.album.artist', 'Justin Bieber'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Christina Aguilera', 'music.artist.genre', 'Teen pop'), ('Christina Aguilera', 'music.artist.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Brandy Norwood', 'music.artist.genre', 'Dance-pop'), ('Love Never Felt So Good', 'music.album.artist', 'Michael Jackson'), ('m.0njw257', 'award.award_honor.award', 'MTV Europe Music Award for Best Push Artist'), ('Country of nationality', 'rdf-schema#domain', 'Person'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Chris Brown', 'people.person.profession', 'Actor'), ('1Club.FM: Power', 'broadcast.content.artist', 'Shaffer Smith'), ('Year', 'type.property.expected_type', 'Date/Time'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Mannie Fresh'), ('Haley James Scott', 'fictional_universe.fictional_character.occupation', 'Singer'), ('Willa Ford', 'people.person.profession', 'Dancer'), ('Tupac Shakur', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njwb81'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Stuart Ford'), (\"Justin Bieber's Believe\", 'film.film.runtime', 'm.0101ftww'), ('Rihanna', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Urban contemporary', 'broadcast.genre.content', 'Hot Wired Radio'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Frank Sinatra', 'people.person.gender', 'Male'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Club Mix)'), ('Sheryl Crow', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Date of birth'), ('Indie rock', 'common.topic.notable_types', 'Musical genre'), ('Usher', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Ja Rule', 'people.person.gender', 'Male'), ('Tyga', 'people.person.nationality', 'United States of America'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.winners', 'm.0yrkc0l'), ('Sean Combs', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'base.icons.icon.icon_genre', 'Teen idol'), ('Pray', 'music.composition.recordings', 'Baby (acoustic)'), ('m.0bnstws', 'common.webpage.in_index', 'Blissful Celebrities'), ('HitzRadio.com', 'broadcast.content.artist', 'Snoop Dogg'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (Neon NiteClub Remix)'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Musical Album', 'freebase.type_profile.equivalent_topic', 'Album'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'Beauty And A Beat'), ('m.0gxnp5m', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Dr. Dre', 'people.person.profession', 'Music executive'), ('Dance music', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('#thatPOWER', 'common.topic.notable_for', 'g.1q3sj3yp4'), ('HitzRadio.com', 'broadcast.content.artist', 'Mis-Teeq'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3fr'), ('m.0y80n25', 'award.award_nomination.nominated_for', 'Heartbreaker'), ('m.0bnstws', 'common.webpage.topic', 'Justin Bieber'), ('Miley Cyrus', 'music.artist.genre', 'Dance-pop'), ('Katy Perry', 'music.artist.genre', 'Synthpop'), ('Kanye West', 'people.person.profession', 'Singer-songwriter'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvwd'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'people.person.profession', 'Artist'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvqf'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fall Out Boy', 'music.artist.genre', 'Rock music'), ('Yves Bole', 'music.artist.album', 'Raindrops'), ('Next to You', 'common.topic.notable_types', 'Musical Recording'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Profession'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Adriano Celentano'), ('Sunshine Radio', 'broadcast.content.language', 'Hungarian language'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ciara'), ('m.0ywsnc9', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Justin Bieber', 'music.featured_artist.recordings', 'Runaway Love (remix)'), ('1.FM Top 40', 'common.topic.notable_types', 'Broadcast Content'), ('Nicki Minaj', 'music.artist.contribution', 'm.0vp8rhw'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_70fx'), ('Fergie', 'people.person.profession', 'Singer-songwriter'), ('P!nk', 'common.topic.notable_types', 'Musical Artist'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('United States of America', 'location.country.administrative_divisions', 'Chicago'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0v30srx'), ('Mannie Fresh', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0y_g556', 'award.award_honor.award_winner', 'Justin Bieber'), ('radioIO Classic RNB', 'broadcast.content.producer', 'Radioio'), ('m.0j219nq', 'music.music_video_performance.music_video_character', 'Singer'), ('Billboard Music Award for Top Digital Media Artist', 'award.award_category.winners', 'm.0njhxd_'), ('Christina Aguilera', 'base.icons.icon.icon_genre', 'Teen idol'), ('Daniel Bedingfield', 'music.artist.label', 'Island Records'), ('Justin Bieber', 'film.actor.film', 'm.0pbzq13'), ('Marvin Isley', 'music.artist.genre', 'Rhythm and blues'), ('m.012m1vf1', 'music.group_membership.role', 'Record producer'), ('Singer', 'theater.theater_character.portrayed_by', 'm.012nh7h_'), ('As Long as You Love Me', 'common.topic.notable_types', 'Composition'), ('Britney Spears', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'people.person.gender', 'Female'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98t5'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Gwen Stefani'), ('Ciara', 'music.artist.genre', 'Hip hop music'), ('m.0vp755b', 'music.recording_contribution.album', 'Beauty and a Beat'), ('Brandy Norwood', 'people.person.profession', 'Model'), ('Robby Stewart', 'fictional_universe.fictional_character.gender', 'Male'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cris Cab'), ('Daniel Bedingfield', 'music.artist.genre', 'Electronic dance music'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0101fvjm', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0gxnnzy', 'celebrities.romantic_relationship.celebrity', 'Selena Gomez'), ('WildFMRadio.com', 'broadcast.content.genre', 'Pop music'), ('Live My Life (Party Rock remix)', 'music.recording.artist', 'Far East Movement'), ('Little Bird', 'music.recording.featured_artists', 'Redfoo'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Clay Aiken'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zbg2kw'), ('Rihanna', 'freebase.notability_hints.notable_for', 'Musical Artist'), ('Dance music', 'music.genre.subgenre', 'Pop music'), ('Jessica Simpson', 'people.person.profession', 'Actor'), ('Barry Weiss', 'business.board_member.leader_of', 'm.0ncpx3v'), ('Juelz Santana', 'people.person.profession', 'Singer'), ('Beyoncé Knowles', 'people.person.languages', 'English Language'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Tupac Shakur', 'people.person.profession', 'Dancer'), ('Twista', 'broadcast.artist.content', '1Club.FM: Power'), ('All That Matters', 'common.topic.notable_types', 'Canonical Version'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0zg8bc1', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fabian', 'people.person.nationality', 'United States of America'), ('1Club.FM: Power', 'broadcast.content.artist', 'Leona Lewis'), ('m.0101fvph', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Justin Bieber'), ('Stephen Melton', 'people.person.gender', 'Male'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5d'), ('NME Award for Worst Album', 'award.award_category.winners', 'm.0z8755b'), ('Keyshia Cole', 'people.person.profession', 'Artist'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Mix 106', 'common.topic.article', 'm.04307gx'), ('Akon', 'people.person.profession', 'Actor'), ('4th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0z0tmyv'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Trick Daddy'), ('My Worlds: The Collection', 'common.topic.notable_types', 'Musical Album'), ('Indie rock', 'common.topic.subject_of', 'Indie rock'), ('Fabolous', 'people.person.profession', 'Record producer'), ('Enrique Iglesias', 'people.person.languages', 'English Language'), ('Britney Spears', 'broadcast.artist.content', 'Hot Wired Radio'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0z8qqh5'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c93b'), ('Lil Jon', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'music.featured_artist.recordings', 'Foreign Remix'), ('m.0gbm3f5', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bigger', 'music.composition.lyricist', 'Kevin Risto'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Classic hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Parachute'), ('Demi Lovato', 'people.person.languages', 'English Language'), ('Gavin DeGraw', 'people.person.gender', 'Male'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Singer'), ('Boyfriend', 'music.recording.canonical_version', 'Boyfriend'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lil Jon'), ('Vanessa Hudgens', 'people.person.profession', 'Actor'), ('Katy Perry: Part of Me', 'film.film.personal_appearances', 'm.0p85jpp'), ('Jayceon Terrell Taylor', 'broadcast.artist.content', 'JellyRadio.com'), ('HitzRadio.com', 'broadcast.content.artist', 'Ja Rule'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'September'), ('2011 Brit Awards', 'award.award_ceremony.awards_presented', 'm.0y803nt'), ('m.0bnstws', 'common.webpage.category', 'Curated Topic'), ('Fergie', 'broadcast.artist.content', 'Sunshine Radio'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.featured_artists', 'Big Sean'), ('Bryan-Michael Cox', 'common.topic.notable_types', 'Record Producer'), ('Les Coulisses des Golden Globes', 'film.film.personal_appearances', 'm.0y5tl39'), ('Bad Day', 'music.composition.composer', \"O'Kelly Isley, Jr.\"), ('Justin Bieber', 'music.composer.compositions', 'Hold Tight'), ('Kelis', 'people.person.gender', 'Female'), ('Jon M. Chu', 'film.director.film', 'Justin Bieber: Never Say Never'), ('Dapo Torimiro', 'music.lyricist.lyrics_written', 'Bigger'), ('Bad 25', 'film.film.genre', 'Music'), ('m.0y5th3r', 'film.personal_film_appearance.film', 'Real Change: Artists for Education'), ('Christina Aguilera', 'people.person.nationality', 'United States of America'), ('Pras', 'broadcast.artist.content', 'HitzRadio.com'), ('Shaffer Smith', 'music.artist.genre', 'Hip hop music'), ('Katy Perry', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('The Pussycat Dolls', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0v90p2b', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Jessie J', 'people.person.profession', 'Artist'), ('Keyshia Cole', 'people.person.profession', 'Singer-songwriter'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Jordan Pruitt'), ('Urban contemporary', 'music.genre.subgenre', 'Quiet Storm'), ('John Mamann', 'people.person.gender', 'Male'), ('m.0p85jpp', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Anastacia', 'music.artist.genre', 'Disco'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Hinder'), ('Alanis Morissette', 'common.topic.notable_types', 'Musical Artist'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Year'), ('1Club.FM: Power', 'broadcast.content.artist', 'Outkast'), ('Contemporary R&B', 'broadcast.genre.content', 'NonStopPlay.com'), ('The Black Eyed Peas', 'music.artist.genre', 'Dance-pop'), ('The Island Def Jam Music Group', 'organization.organization.previous_names', 'm.05sp3_n'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c927'), ('1Club.FM: Channel One', 'common.topic.article', 'm.04307hl'), ('LeAnn Rimes', 'music.artist.genre', 'Dance music'), ('Willa Ford', 'music.artist.genre', 'Pop music'), ('Runaway Love', 'common.topic.notable_types', 'Musical Recording'), ('Nathan Lanier', 'common.topic.notable_types', 'Musical Artist'), ('m.0y_g556', 'award.award_honor.award', 'Shorty Award for Celebrity'), ('Marvin Isley', 'people.person.gender', 'Male'), ('m.0101fsyr', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('Lolly', 'music.recording.song', 'Lolly'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'L.A. Reid'), ('Maroon 5', 'broadcast.artist.content', '1.FM Top 40'), ('Never Say Never: The Remixes', 'music.album.primary_release', 'Never Say Never: The Remixes'), ('First Dance', 'music.composition.composer', 'Ryan Lovette'), ('1.FM Top 40', 'broadcast.content.artist', 'Justin Bieber'), ('Justin Timberlake', 'people.person.profession', 'Musician'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsy'), ('m.0v30srx', 'award.award_nomination.nominated_for', 'Believe'), ('m.0_x3kvk', 'award.award_nomination.award_nominee', 'R. Kelly'), ('Hip hop music', 'common.topic.notable_types', 'Musical genre'), ('Spouse', 'type.property.reverse_property', 'Spouse (or domestic partner)'), ('#thatPOWER', 'common.topic.notable_for', 'g.12h2w5tf4'), ('The Notorious B.I.G.', 'common.topic.notable_types', 'Musical Artist'), ('Juno Awards of 2011', 'award.award_ceremony.awards_presented', 'm.0gwhmhm'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Leonardo DiCaprio', 'base.icons.icon.icon_genre', 'Teen idol'), ('Teen Choice Award for Choice Male Hottie', 'award.award_category.winners', 'm.0yrktlv'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsj'), ('Fall Out Boy', 'music.artist.label', 'The Island Def Jam Music Group'), ('Fabolous', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Boys Like Girls'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Katy Perry'), ('Katy Perry: Part of Me', 'film.film.directed_by', 'Jane Lipsitz'), ('Blogger', 'people.profession.people_with_this_profession', 'Yves Bole'), ('m.0101ft56', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Teyana', 'people.person.nationality', 'United States of America'), ('Rita Ora', 'people.person.profession', 'Singer-songwriter'), ('All Around the World (acoustic version)', 'music.recording.song', 'All Around The World'), ('Trick Daddy', 'people.person.profession', 'Actor'), ('m.0z8qx3w', 'award.award_honor.award_winner', 'Justin Bieber'), ('Frank Ocean', 'common.topic.notable_types', 'Musical Artist'), ('m.0yrhrwc', 'award.award_honor.award', 'MTV Video Music Award Japan for Best New Artist'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv9w'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Alanis Morissette', 'people.person.gender', 'Female'), ('PYD', 'music.recording.song', 'PYD'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Urban contemporary'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Ronald Isley', 'people.person.profession', 'Musician'), ('Lupe Fiasco', 'people.person.place_of_birth', 'Chicago'), ('My World 2.0', 'music.album.genre', 'Dance-pop'), ('JoJo', 'people.person.profession', 'Singer'), ('Next to You', 'common.topic.notable_for', 'g.1yg9k0xh7'), ('Shaffer Smith', 'broadcast.artist.content', 'HitzRadio.com'), ('Chris Brown', 'broadcast.artist.content', 'Hot Wired Radio'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Tupac Shakur'), ('Jeremy Bieber', 'common.topic.notable_for', 'g.1258wnf9x'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c91k'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sean Combs'), ('m.0tkqqgg', 'award.award_nomination.nominated_for', 'My World'), ('Cory Gunz', 'people.person.nationality', 'United States of America'), ('1Club.FM: Power', 'broadcast.content.artist', 'Britney Spears'), ('United States of America', 'location.country.currency_used', 'United States Dollar'), ('Men in Black 3', 'film.film.language', 'English Language'), ('Nelly', 'broadcast.artist.content', '.977 The Hits Channel'), ('Linkin Park', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber: Just Getting Started', 'common.topic.notable_types', 'Book'), ('Ludacris', 'music.artist.genre', 'Hip hop music'), ('m.0sgkw8v', 'award.award_honor.ceremony', \"2012 Kids' Choice Awards\"), ('DMX', 'broadcast.artist.content', 'JellyRadio.com'), ('Recovery', 'music.composition.composer', 'Mark Hill'), ('Justin Bieber', 'music.featured_artist.recordings', 'Little Bird'), ('American Music Award for Artist of the Year', 'award.award_category.winners', 'm.0ndc3_1'), ('Leif Garrett', 'people.person.gender', 'Male'), ('Linkin Park', 'music.artist.genre', 'Hip hop music'), ('m.0z1jn32', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.song', 'Beauty And A Beat'), ('Ronald Isley', 'music.artist.genre', 'Contemporary R&B'), ('#thatPower', 'common.topic.notable_types', 'Musical Recording'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Jessica Simpson'), ('Savan Kotecha', 'people.person.gender', 'Male'), ('m.0njwqrb', 'award.award_honor.award_winner', 'Justin Bieber'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Tricky Stewart', 'music.composer.compositions', 'Baby'), ('The Pussycat Dolls', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jeremy Bieber', 'people.person.children', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'film.film.directed_by', 'Jon M. Chu'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Gender'), ('Max Martin', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'common.topic.image', 'Justin bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Avril Lavigne'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2d0'), ('Journals', 'music.album.releases', 'Journals'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Rihanna'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronski Speed', 'music.artist.genre', 'Trance music'), ('Will Smith', 'broadcast.artist.content', 'radioIO Todays POP'), ('Alicia Keys', 'broadcast.artist.content', 'radioIO RNB Mix'), (\"Turn to You (Mother's Day Dedication)\", 'music.recording.song', \"Turn to You (Mother's Day Dedication)\"), ('Nelly Furtado', 'music.artist.genre', 'Rhythm and blues'), ('New Kids on the Block', 'music.artist.genre', 'Pop music'), ('Right Here', 'music.recording.featured_artists', 'Drake'), ('Aaliyah', 'people.person.gender', 'Female'), ('m.05sp41b', 'business.company_name_change.company', 'The Island Def Jam Music Group'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Kanye West'), ('One Less Lonely Girl', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c933'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9h3k'), ('Twista', 'music.artist.genre', 'Rhythm and blues'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), (\"Justin Bieber's Believe\", 'film.film.distributors', 'm.012f2m86'), ('Driving under the influence', 'celebrities.reason_for_arrest.celebrities_charged_or_arrested', 'm.0_cyzs_'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Juvenile', 'freebase.valuenotation.has_value', 'Parents'), ('Snoop Dogg', 'broadcast.artist.content', 'FLOW 103'), ('Confident', 'common.topic.notable_for', 'g.1yp3dc66c'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Clay Aiken', 'people.person.nationality', 'United States of America'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Blackstreet'), ('Miley Cyrus', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Sia Furler', 'broadcast.artist.content', '1.FM Top 40'), ('m.0101gx29', 'people.marriage.spouse', 'Pattie Mallette'), ('Confident', 'music.recording.song', 'Confident'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Barcera'), ('Christina Aguilera', 'music.artist.genre', 'Pop music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Pearl Jam', 'freebase.valuenotation.is_reviewed', 'Official website'), ('1.FM Top 40', 'broadcast.content.artist', 'Gym Class Heroes'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO Todays RNB'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Thought of You', 'music.recording.song', 'Thought Of You'), ('Recovery', 'common.topic.notable_types', 'Composition'), ('Big R Radio - Top 40 Hits', 'common.topic.notable_for', 'g.125b5lghf'), ('m.0z83x9l', 'award.award_honor.award', 'NME Award for Villain of the Year'), ('Twista', 'music.artist.genre', 'Hip hop music'), ('m.01053qzf', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k02'), ('Ludacris', 'people.person.gender', 'Male'), ('Record producer', 'music.performance_role.guest_performances', 'm.0h3_p71'), ('Dr. Dre', 'music.artist.genre', 'Hip hop music'), ('Vocals', 'music.instrument.instrumentalists', 'Stephen Melton'), ('First Dance', 'music.composition.lyricist', 'Ryan Lovette'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftnq'), ('My Worlds Acoustic', 'common.topic.article', 'm.0fph3j0'), (\"Destiny's Child\", 'music.artist.genre', 'Teen pop'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0njvvj_'), ('The Notorious B.I.G.', 'people.person.nationality', 'United States of America'), ('KooL CrAzE', 'music.artist.genre', 'Pop music'), ('Akon', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0yrhmll', 'award.award_honor.award', 'MTV Europe Music Award for Best North American Act'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Miley Cyrus'), ('Hip hop music', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Yves Bole', 'people.person.languages', 'Spanish Language'), ('m.0j593k1', 'film.film_regional_release_date.film', 'Justin Bieber: Never Say Never'), ('Canada', 'location.location.time_zones', 'Eastern Time Zone'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Katy Perry'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c91t'), ('PowerHitz', 'broadcast.content.artist', '50 Cent'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Shaggy', 'broadcast.artist.content', '1Club.FM: Power'), ('Ernie Isley', 'music.artist.genre', 'Rock music'), ('Bryan-Michael Cox', 'people.person.profession', 'Record producer'), ('Amerie', 'people.person.nationality', 'United States of America'), ('m.0yqflrk', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0nfnx_3'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Will Smith', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Victoria Justice', 'music.artist.genre', 'Synthpop'), ('Justin Bieber', 'common.topic.article', 'm.06w2sn9'), ('My World', 'common.topic.image', 'My World'), ('Urban contemporary', 'broadcast.genre.content', '181-beat'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Year'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Willa Ford', 'music.artist.origin', 'Tampa'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Rihanna'), ('Madonna', 'broadcast.artist.content', 'radioIO Todays POP'), ('RedOne', 'people.person.profession', 'Music executive'), ('Enrique Iglesias', 'broadcast.artist.content', 'radioIO Todays POP'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber Lyrics', 'common.resource.annotations', 'm.0d_hbgr'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Sisqó'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten remix)'), ('Jon M. Chu', 'people.person.gender', 'Male'), ('m.0njvs9s', 'award.award_honor.honored_for', 'That Should Be Me'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.featured_artists', 'Big Sean'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Believe Acoustic', 'music.album.album_content_type', 'Remix album'), ('This is Justin Bieber', 'common.topic.notable_types', 'Film'), ('Gwen Stefani', 'broadcast.artist.content', '.977 The Hits Channel'), ('Kelis', 'music.artist.genre', 'Electronic music'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Evanescence'), ('R. Kelly', 'people.person.gender', 'Male'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Year'), ('Madonna', 'music.artist.genre', 'Dance music'), ('Country', 'freebase.type_hints.included_types', 'Location'), ('As Long as You Love Me', 'music.album.primary_release', 'As Long as You Love Me'), ('Lady Gaga', 'music.artist.genre', 'Electronic music'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Rune Reilly Kølsch'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Don Henley', 'broadcast.artist.content', '1.FM Top 40'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vb6hhj', 'award.award_honor.award', 'Billboard Music Milestone Award'), ('#thatPOWER', 'music.recording.tracks', 'ThatPower'), ('Hold Tight', 'music.recording.song', 'Hold Tight'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Bad Day', 'music.recording.artist', 'Justin Bieber'), ('LeAnn Rimes', 'music.artist.genre', 'Rhythm and blues'), ('Wait For a Minute', 'music.album.primary_release', 'Wait For a Minute'), ('William Orbit', 'music.artist.genre', 'Electronic music'), ('Ontario', 'location.location.containedby', 'Canada'), ('m.012bm2d0', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Record producer', 'common.topic.webpage', 'm.09wyfdf'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ashlee Simpson', 'music.artist.genre', 'Dance-pop'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Lupe Fiasco'), ('m.0njw444', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Christina Milian', 'music.artist.genre', 'Hip hop music'), ('Heartbreaker', 'music.album.release_type', 'Single'), ('Boyfriend', 'music.single.versions', 'Boyfriend'), ('Kelly Clarkson', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Beauty and a Beat (Remixes)', 'music.album.contributor', 'm.0vp8rhw'), ('m.0tjyljn', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Elvis Presley', 'music.artist.genre', 'Rock music'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Contemporary R&B'), ('Frank Ocean', 'music.artist.label', 'The Island Def Jam Music Group'), ('Katy Perry: Part of Me', 'film.film.music', 'Deborah Lurie'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Montell Jordan', 'broadcast.artist.content', 'radioIO Todays POP'), ('Chris Jasper', 'people.person.profession', 'Musician'), ('Green Day', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jazmyn Bieber', 'people.person.sibling_s', 'm.0gxnnwc'), ('Pattie Mallette', 'film.person_or_entity_appearing_in_film.films', 'm.0gz0fr6'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_6_ww'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc0sf'), ('Ray J', 'people.person.profession', 'Actor'), ('Rhythm and blues', 'common.topic.notable_types', 'Musical genre'), ('Baby (acoustic)', 'music.recording.canonical_version', 'Pray'), ('The Isley Brothers', 'common.topic.notable_types', 'Musical Artist'), ('All Around The World', 'music.composition.composer', 'Ludacris'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('HitzRadio.com', 'broadcast.content.artist', 'Kelly Clarkson'), ('Anastacia', 'broadcast.artist.content', '1.FM Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_3bt'), ('m.0t_l7mp', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Estelle', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Adrienne Bailon'), ('Eenie Meenie', 'music.recording.featured_artists', 'Sean Kingston'), ('Hip hop music', 'broadcast.genre.content', 'PowerHitz'), ('Verse Simmonds', 'people.person.gender', 'Male'), ('m.0gxnny8', 'people.place_lived.location', 'Canada'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Linkin Park'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1.FM Top 40', 'broadcast.content.genre', 'Pop music'), ('2013 Billboard Music Awards', 'time.event.people_involved', 'Shaffer Smith'), ('Sean Kingston', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Live My Life', 'common.topic.notable_for', 'g.126tdsljg'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grm8q'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Jayceon Terrell Taylor', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'broadcast.artist.content', 'PowerHitz'), ('Juvenile', 'people.person.profession', 'Actor'), ('Love Never Felt So Good', 'music.composition.language', 'English Language'), ('m.0h3_p71', 'music.recording_contribution.performance_role', 'Record producer'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.song', 'Beauty And A Beat'), ('RedOne', 'common.topic.notable_types', 'Record Producer'), ('Rudolph Valentino', 'people.person.profession', 'Actor'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('m.0gxnp5x', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'music.featured_artist.recordings', 'Eenie Meenie'), ('Janet Jackson', 'music.artist.genre', 'Dance music'), ('Somebody To Love', 'music.recording.song', 'Somebody to Love'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_729v'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.0yqsnwz'), ('Pray', 'music.album.releases', 'Pray'), ('1Club.FM: Power', 'broadcast.content.artist', 'K-Ci & JoJo'), ('Lil Wayne', 'music.artist.genre', 'Contemporary R&B'), ('Whitney Houston', 'common.topic.notable_types', 'Musical Artist'), (\"Destiny's Child\", 'broadcast.artist.content', '1Club.FM: Channel One'), ('Taylor Swift', 'people.person.profession', 'Musician'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Nelly Furtado', 'people.person.profession', 'Musician'), ('1Club.FM: Power', 'broadcast.content.artist', 'Cherish'), ('1Club.FM: Power', 'broadcast.content.artist', 'Katy Perry'), ('Coldplay', 'music.artist.genre', 'Pop music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Maroon 5'), ('Blackstreet', 'music.artist.genre', 'Rhythm and blues'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Date of birth'), ('R. Kelly', 'music.artist.genre', 'Pop music'), ('Lupe Fiasco', 'people.person.profession', 'Record producer'), ('Bad Day', 'music.composition.composer', 'Dominic Jordan'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'OneRepublic'), ('Kuk Harrell', 'people.person.gender', 'Male'), ('Donna Summer', 'music.artist.genre', 'Pop music'), ('GotRadio - RnB Classics', 'common.topic.notable_types', 'Broadcast Content'), ('Kanye West', 'broadcast.artist.content', 'HitzRadio.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Flo Rida'), ('Teen idol', 'base.icons.icon_genre.icons', 'Johnny Crawford'), ('Beauty and a Beat (Remixes)', 'music.album.release_type', 'EP'), ('m.0ng_j6d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Mistletoe', 'common.topic.notable_types', 'Musical Album'), ('Pray', 'music.recording.song', 'Pray'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'broadcast.artist.content', '1Club.FM: Power'), ('Acoustic music', 'common.topic.notable_types', 'Musical genre'), ('m.0yrkgd6', 'award.award_honor.award', 'Teen Choice Award for Choice Summer Music Star: Male'), ('Johntá Austin', 'music.artist.genre', 'Contemporary R&B'), ('Sean Combs', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Ricky Nelson', 'music.artist.genre', 'Rock music'), ('HitzRadio.com', 'broadcast.content.artist', 'Justin Bieber'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO Todays POP'), ('My Worlds', 'common.topic.notable_for', 'g.1259t8kb3'), ('Justin Timberlake', 'music.artist.genre', 'Dance music'), ('Aaliyah', 'music.artist.genre', 'Pop music'), ('Kelly Clarkson', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.09wyfdf', 'common.webpage.topic', 'Record producer'), ('Right Here', 'music.album.album_content_type', 'Studio album'), ('Chingy', 'freebase.valuenotation.has_value', 'Parents'), ('Juicy J', 'music.featured_artist.albums', 'Lolly'), ('Composition', 'type.type.properties', 'Composer'), ('#thatPower', 'music.composition.recorded_as_album', '#thatPOWER'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Journals', 'music.album.genre', 'Contemporary R&B'), ('Believe', 'music.album.releases', 'Believe'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101fvkw', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sia Furler', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3c3'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk18w'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Leif Garrett', 'base.icons.icon.icon_genre', 'Teen idol'), ('Profession', 'type.property.reverse_property', 'People With This Profession'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk4gn'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO Classic RNB'), ('Alicia Keys', 'music.artist.genre', 'Contemporary R&B'), ('Men in Black 3', 'film.film.country', 'United States of America'), ('Whitney Houston', 'people.person.profession', 'Film Producer'), ('The Pussycat Dolls', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'common.topic.webpage', 'm.03hs4ny'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Rihanna'), ('PYD', 'common.topic.notable_for', 'g.1yw57ynz4'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Green Day', 'music.artist.genre', 'Rock music'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Dance music'), ('R. Kelly', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Official website'), ('.977 The Hits Channel', 'broadcast.content.artist', 'TLC'), ('m.0101fvf2', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njdns_'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkrj4'), ('Hot Wired Radio', 'broadcast.content.artist', \"Plain White T's\"), ('Jennifer Lopez', 'people.person.profession', 'Singer'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Boyfriend', 'music.recording.releases', 'Believe'), ('Boyfriend', 'award.award_nominated_work.award_nominations', 'm.0z84kwx'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('As Long as You Love Me (album version)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Tricky Stewart', 'music.artist.genre', 'Pop music'), ('Caitlin Beadles', 'people.person.place_of_birth', 'Canada'), ('Fergie', 'music.artist.genre', 'Hip hop music'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0_cz5l3', 'celebrities.legal_entanglement.offense', 'Driving under the influence'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 128kbps Stream'), ('Ronald Isley', 'music.artist.genre', 'Rock music'), ('Tupac Shakur', 'broadcast.artist.content', 'JellyRadio.com'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kanye West', 'music.artist.genre', 'Hip hop music'), ('Christina Aguilera', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Justin Bieber', 'music.composer.compositions', \"Turn to You (Mother's Day Dedication)\"), ('Savan Kotecha', 'music.artist.genre', 'Pop music'), ('Fabian', 'people.person.profession', 'Singer'), ('Hikaru Utada', 'common.topic.notable_types', 'Musical Artist'), ('Jay-Z', 'music.artist.genre', 'Hip hop music'), ('The Island Def Jam Music Group', 'award.award_nominee.award_nominations', 'm.0ws63cr'), ('Terius Nash', 'broadcast.artist.content', 'HitzRadio.com'), ('Rob Thomas', 'music.artist.genre', 'Pop music'), ('Tampa', 'location.location.time_zones', 'Eastern Time Zone'), ('Live My Life (Jaywalker remix)', 'music.recording.song', 'Live My Life'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten radio)'), ('Hit-Boy', 'people.person.gender', 'Male'), ('Justin Bieber', 'internet.blogger.blog', 'justinbieber'), ('Zendaya: Behind the Scenes', 'film.film.language', 'English Language'), ('1Club.FM: V101', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long as You Love Me', 'music.composition.recorded_as_album', 'As Long as You Love Me'), ('Taylor Swift', 'celebrities.celebrity.celebrity_friends', 'm.0_grqb8'), ('Hot Wired Radio', 'broadcast.content.artist', 'Carrie Underwood'), ('m.0z85qxq', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Savan Kotecha', 'music.composer.compositions', 'Beauty And A Beat'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Justin Timberlake'), ('Jennifer Lopez', 'broadcast.artist.content', 'radioIO Todays POP'), ('John Mamann', 'music.artist.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Rudolph Valentino'), ('m.0jsmvv5', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('HitzRadio.com', 'broadcast.content.artist', 'Lil Rob'), ('Hot Wired Radio', 'broadcast.content.artist', 'The All-American Rejects'), ('m.0sgk_cw', 'award.award_honor.ceremony', \"2011 Kids' Choice Awards\"), ('Ronald Isley', 'music.artist.genre', 'Hip hop music'), ('Official website', 'type.property.expected_type', 'URI'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.song', 'As Long as You Love Me'), ('J. Holiday', 'music.artist.genre', 'Hip hop music'), ('Kings of Leon', 'common.topic.notable_types', 'Musical Artist'), ('My World', 'common.topic.notable_for', 'g.125g8b_3v'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj_1'), ('Pras', 'people.person.gender', 'Male'), ('Ashley Tisdale', 'base.icons.icon.icon_genre', 'Teen idol'), ('Singer-songwriter', 'base.musicpf.genre.artist', 'Yves Bole'), ('m.0gxnnwc', 'people.sibling_relationship.sibling', 'Justin Bieber'), ('m.0_grqb8', 'celebrities.friendship.friend', 'Justin Bieber'), ('P!nk', 'broadcast.artist.content', 'radioIO Todays POP'), ('Shorty Award for Celebrity', 'award.award_category.winners', 'm.0ywvh8k'), ('HitzRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Lolly', 'common.topic.notable_for', 'g.12v_h9661'), ('Live My Life (Jaywalker remix)', 'common.topic.notable_for', 'g.1yg9hnqh7'), ('Rihanna', 'music.artist.genre', 'Eurodance'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Height'), ('Iggy Azalea', 'music.artist.genre', 'Hip hop music'), ('The Roots', 'music.artist.genre', 'Rock music'), ('Nathan Lanier', 'people.person.profession', 'Musician'), ('Ginuwine', 'music.artist.origin', 'United States of America'), ('Beyoncé Knowles', 'people.person.profession', 'Musician'), ('Bigger', 'common.topic.notable_types', 'Canonical Version'), ('6 Tre G', 'music.artist.genre', 'Hip hop music'), ('Jaxon Bieber', 'common.topic.notable_types', 'Person'), ('Beyoncé Knowles', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('m.0gbm3d_', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('m.0_xlw5f', 'award.award_nomination.nominated_for', 'Wait for a Minute'), ('m.0gc_9w6', 'common.webpage.category', 'Video Stream Host Page'), ('m.0nfnx_3', 'film.personal_film_appearance.person', 'Justin Bieber'), ('We Are the World 25 for Haiti', 'music.album.contributor', 'm.0vpggkk'), ('Dapo Torimiro', 'people.person.profession', 'Singer'), ('Fabolous', 'people.person.nationality', 'United States of America'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'EPMD'), ('Christina Aguilera', 'people.person.profession', 'Record producer'), ('Taylor Swift', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1w0t'), ('First Dance', 'common.topic.article', 'm.0g5q9l1'), ('Chris Brown', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Mary J. Blige', 'music.artist.genre', 'Rhythm and blues'), ('Rihanna', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('FLOW 103', 'broadcast.content.artist', 'Akon'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw4z2'), ('Confident', 'common.topic.notable_for', 'g.1yp3bsmrd'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z1ndbl'), ('m.0101ft2v', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flo Rida'), ('Young Artists for Haiti', 'common.topic.notable_types', 'Musical Artist'), ('Jennifer Lopez', 'people.person.profession', 'Actor'), ('Jordin Sparks', 'music.artist.genre', 'Pop music'), ('Ricky Nelson', 'music.artist.genre', 'Pop music'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjgrzc'), ('HitzRadio.com', 'common.topic.image', 'HitzRadio.com'), ('Boyfriend', 'music.composition.composer', 'Mason Levy'), ('Boyfriend (Dada Life remix)', 'music.recording.song', 'Boyfriend'), ('Tampa', 'location.hud_county_place.place', 'Tampa'), ('Jessie J', 'music.artist.genre', 'Contemporary R&B'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Colbie Caillat', 'music.artist.genre', 'Acoustic music'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Year'), ('Snoop Dogg', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber', 'music.featured_artist.recordings', 'Beautiful'), ('Justin Bieber', 'music.artist.origin', 'Stratford'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO RNB Mix'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Year'), ('Date of birth', 'rdf-schema#domain', 'Person'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'TLC'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Chris Brown'), ('Pop music', 'base.musicpf.genre.artist', 'Yves Bole'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me (album version)'), ('FLOW 103', 'broadcast.content.artist', 'Baby Bash'), ('m.0_srv2b', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Jordan Francis', 'people.person.profession', 'Actor'), ('Christina Aguilera', 'music.artist.genre', 'Rock music'), ('Justin Timberlake', 'people.person.profession', 'Singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Taylor Swift'), ('Justin Bieber', 'people.person.profession', 'Dancer'), ('m.0z2dr9y', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'International'), ('My World', 'music.album.releases', 'My World'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.canonical_version', 'Boyfriend'), ('Dance music', 'broadcast.genre.content', '1Club.FM: 80s (Pop)'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Spouse (or domestic partner)'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.artist', 'Justin Bieber'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Nickelback'), ('Big R Radio - Top 40 Hits', 'broadcast.content.producer', 'Big R Radio Network'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Fabolous', 'broadcast.artist.content', '.977 The Hits Channel'), ('Brandy Norwood', 'music.artist.genre', 'Pop music'), ('Chicago', 'location.administrative_division.country', 'United States of America'), ('Khalil', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Bryan Adams', 'people.person.profession', 'Record producer'), ('Cory Gunz', 'music.artist.genre', 'Hip hop music'), ('Nasri', 'music.composer.compositions', 'All Around The World'), ('Person or entity appearing in film', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Remix)'), ('m.0y7y53r', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0z8t2dy', 'award.award_nomination.award', 'NME Award for Worst Album'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Lolly', 'music.recording.song', 'Lolly'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Power'), ('Rob Thomas', 'music.artist.genre', 'Rock music'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chris Brown'), ('Barry Weiss', 'people.person.gender', 'Male'), ('CL', 'music.artist.genre', 'Pop music'), ('FLOW 103', 'broadcast.content.artist', 'The Black Eyed Peas'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mat Silver & Tony Burt'), ('My World', 'music.album.releases', 'My World'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.song', 'As Long as You Love Me'), ('Hot Wired Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Ice Cube', 'people.person.profession', 'Actor'), ('Boyfriend', 'music.single.versions', 'Boyfriend (Neon NiteClub Remix)'), ('m.0_xlw5f', 'award.award_nomination.award_nominee', 'Tyga'), ('Boyfriend', 'common.topic.article', 'm.0j64n8h'), ('Beauty and a Beat', 'music.album.genre', 'Electronic dance music'), ('Leif Garrett', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmxj'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0tkqqgg'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('David Nicksay', 'film.producer.films_executive_produced', 'Justin Bieber: Never Say Never'), ('Musical Artist', 'type.type.expected_by', 'Artist'), ('Pras', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.artist.album', 'Heartbreaker'), ('Janet Jackson', 'music.artist.genre', 'Hip hop music'), ('m.0101ft56', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('radioIO Todays RNB', 'broadcast.content.genre', 'Rhythm and blues'), ('Ciara', 'broadcast.artist.content', '.977 The Hits Channel'), ('Demi Lovato', 'music.artist.genre', 'Rock music'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('All That Matters', 'common.topic.notable_for', 'g.1yl5vjhm6'), ('Colbie Caillat', 'people.person.nationality', 'United States of America'), ('m.0n4rmg7', 'award.award_honor.honored_for', 'U Smile'), ('FLOW 103', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Height'), ('My Worlds Acoustic', 'music.album.release_type', 'Album'), ('m.0v90skf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Justin Bieber: First Step 2 Forever', 'common.topic.notable_for', 'g.1yl5jc8_8'), ('m.0yrk18w', 'award.award_honor.award_winner', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Crazy Town'), ('Lady Antebellum', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Love Never Felt So Good', 'music.album.artist', 'Justin Bieber'), ('Avril Lavigne', 'music.artist.genre', 'Teen pop'), ('Ray J', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miami Beach', 'location.hud_county_place.place', 'Miami Beach'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Somebody to Love (remix)', 'music.album.artist', 'Justin Bieber'), ('Adrienne Bailon', 'music.artist.genre', 'Contemporary R&B'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Year'), ('m.0vp7cl4', 'music.recording_contribution.album', 'Somebody to Love (remix)'), ('Shaun Cassidy', 'people.person.profession', 'Film Producer'), ('Cris Cab', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0yqsnwz', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('Christian Beadles', 'people.person.gender', 'Male'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('m.0z3tqqt', 'award.award_nomination.nominated_for', 'justinbieber'), ('WildFMRadio.com', 'broadcast.content.artist', 'Ludacris'), ('Jay-Z', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Trick Daddy', 'music.artist.origin', 'Miami'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Baby Bash'), ('Jordan Francis', 'music.artist.genre', 'Pop music'), ('m.0pc67ly', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0njgyk4', 'award.award_honor.honored_for', 'My World 2.0'), ('Whitney Houston', 'influence.influence_node.influenced', 'Lady Gaga'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Raekwon', 'music.artist.contribution', 'm.0vp7m_x'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Kanye West'), ('Kuk Harrell', 'common.topic.notable_types', 'Record Producer'), ('Urban contemporary', 'broadcast.genre.content', 'HitzRadio.com'), ('Justin Bieber', 'music.composer.compositions', 'Somebody to Love'), ('Blu Cantrell', 'people.person.nationality', 'United States of America'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Coldplay', 'common.topic.notable_types', 'Musical Artist'), ('Kid Cudi', 'people.person.profession', 'Actor'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Official website'), ('HitzRadio.com', 'broadcast.content.artist', 'Flobots'), ('Soulja Boy', 'people.person.profession', 'Actor'), ('Record producer', 'base.sounds.audio_recording_role.recordings', 'm.09jx30f'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mariah Carey'), ('The Pussycat Dolls', 'music.artist.genre', 'Pop music'), ('Pattie Mallette', 'common.topic.notable_types', 'Author'), ('Recovery', 'music.composition.composer', 'Dominic Jordan'), ('New Kids on the Block', 'music.artist.genre', 'Hip hop music'), ('Taylor Swift', 'music.artist.genre', 'Teen pop'), ('HitzRadio.com', 'broadcast.content.artist', 'September'), ('Singer', 'common.topic.notable_types', 'Profession'), ('Alanis Morissette', 'people.person.profession', 'Writer'), ('Aaliyah', 'music.artist.genre', 'Dance-pop'), ('R. Kelly', 'music.artist.origin', 'Chicago'), ('Baby', 'music.composition.recorded_as_album', 'Baby'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Yves Bole'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'people.profession.specialization_of', 'Musician'), ('RBMG Records', 'common.topic.notable_types', 'Record label'), ('Michael Jackson', 'broadcast.artist.content', 'GotRadio - RnB Classics'), ('Katy Perry', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber Fan Club', 'common.resource.annotations', 'm.0dj34q5'), ('Annette Funicello', 'people.person.nationality', 'United States of America'), ('Usher', 'common.topic.notable_types', 'Musical Artist'), ('#thatPOWER', 'common.topic.notable_for', 'g.1yg9cvthd'), ('Jeremy Bieber', 'people.person.children', 'Jaxon Bieber'), ('m.0101fv7x', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0z340zt', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_72js'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Bigger'), ('m.01053qzf', 'film.personal_film_appearance.person', 'Jeremy Bieber'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0yrhmll'), ('Heartbreaker', 'award.award_nominated_work.award_nominations', 'm.0y80n25'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Rihanna'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Taylor Swift'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ludacris'), ('MTV Europe Music Award for Best Canadian Act', 'award.award_category.winners', 'm.0yqfny6'), ('Timbaland', 'people.person.nationality', 'United States of America'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (acoustic version)'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Judy Garland', 'people.person.gender', 'Female'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Diplo', 'music.artist.genre', 'Hip hop music'), ('Ice Cube', 'people.person.profession', 'Record producer'), ('Bryan-Michael Cox', 'people.person.gender', 'Male'), ('My World', 'music.album.genre', 'Dance-pop'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074ns9'), ('School Boy Records', 'music.record_label.artist', 'CL'), ('Eenie Meenie', 'common.topic.notable_for', 'g.1yp3c7ts1'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrk4gn'), ('Taylor Swift', 'freebase.valuenotation.has_no_value', 'Children'), ('HitzRadio.com', 'broadcast.content.artist', 'Baby Bash'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Demi Lovato', 'people.person.gender', 'Female'), ('Pras', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Usher', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'people.person.profession', 'Singer'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Area codes 519 and 226', 'location.location.contains', 'London'), ('Lil Wayne', 'people.person.gender', 'Male'), (\"2013 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkrj4'), ('Chicago', 'location.location.containedby', 'United States of America'), ('Usher', 'broadcast.artist.content', '.977 The Hits Channel'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Alanis Morissette'), ('m.0_cz5l3', 'celebrities.legal_entanglement.celebrity', 'Justin Bieber'), ('Film', 'freebase.type_profile.strict_included_types', 'Topic'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0j2gfc1'), ('m.0gj1yzm', 'tv.tv_guest_role.episodes_appeared_in', 'Series 1, Show 4'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kanye West'), ('Next to You', 'music.recording.featured_artists', 'Justin Bieber'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Enrique Iglesias', 'people.person.nationality', 'United States of America'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.song', 'Boyfriend'), ('Anastacia', 'music.artist.origin', 'Chicago'), ('Fergie', 'people.person.profession', 'Musician'), ('Usher', 'music.artist.contribution', 'm.0vp7cl4'), ('Justin Bieber', 'people.person.places_lived', 'm.0kt8f87'), ('Ronald Isley', 'music.artist.genre', 'Rhythm and blues'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Dance music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z87d3n'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Nicki Minaj', 'people.person.profession', 'Musician'), ('Chris Jasper', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft56'), ('Favorite Girl', 'common.topic.notable_for', 'g.12h323s38'), ('Die in Your Arms', 'common.topic.notable_for', 'g.1yh9szjwm'), ('Hip hop music', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('Singer', 'common.topic.notable_for', 'g.12557cdvn'), ('Lady Gaga', 'people.person.profession', 'Record producer'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All Bad', 'music.recording.artist', 'Justin Bieber'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhxzc'), ('m.0gbcs16', 'tv.tv_guest_role.episodes_appeared_in', 'August 20, 2010'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0tk76mf'), ('Contemporary R&B', 'broadcast.genre.content', 'WildFMRadio.com'), ('Verse Simmonds', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Eenie Meenie', 'music.album.artist', 'Sean Kingston'), ('Jay-Z', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0_cyzs_', 'celebrities.legal_entanglement.celebrity', 'Justin Bieber'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Teen idol', 'common.topic.webpage', 'm.09wnjlc'), ('Annette Funicello', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Country', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_k21'), ('Lupe Fiasco', 'common.topic.notable_types', 'Musical Artist'), ('Drake', 'music.featured_artist.albums', 'Right Here'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'AnnaGrace'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sheryl Crow', 'people.person.profession', 'Actor'), ('Pop music', 'music.genre.subgenre', 'Synthpop'), ('FLOW 103', 'broadcast.content.genre', 'Contemporary R&B'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Jay Cassidy'), ('Dan Cutforth', 'film.director.film', 'Katy Perry: Part of Me'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjpm'), ('m.0_xlw5f', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0_sxby0', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0wjgqck', 'award.award_honor.award', 'Teen Choice Award for Choice Single: Male Artist'), ('Selena Gomez', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('Lady Gaga', 'people.person.profession', 'Actor'), ('GotRadio - RnB Classics', 'broadcast.content.language', 'English Language'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Sir Mix-a-Lot', 'people.person.profession', 'Musician'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Will Smith', 'people.person.nationality', 'United States of America'), ('The Notorious B.I.G.', 'broadcast.artist.content', '.977 The Hits Channel'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Contemporary R&B'), ('m.0101fvhv', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Christian Beadles', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsj'), ('m.0gbm3fr', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Chicago', 'common.topic.notable_types', 'City/Town/Village'), ('HitzRadio.com', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Justin Bieber', 'music.artist.album', 'One Less Lonely Girl'), ('My Worlds', 'music.album.release_type', 'Album'), ('CL', 'music.artist.genre', 'Rhythm and blues'), ('CL', 'people.person.profession', 'Singer'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Twista', 'broadcast.artist.content', '181-beat'), ('Dr. Dre', 'people.person.gender', 'Male'), ('Christina Aguilera', 'music.artist.genre', 'Synthpop'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO Todays RNB'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Brandy Norwood', 'people.person.profession', 'Film Producer'), ('Live My Life', 'music.composition.composer', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Place of birth', 'type.property.expected_type', 'Location'), ('Kuk Harrell', 'people.person.nationality', 'United States of America'), ('Beauty and a Beat', 'music.album.genre', 'Synthpop'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0sxhq3d'), ('Shaun Cassidy', 'base.icons.icon.icon_genre', 'Teen idol'), ('Usher', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Montell Jordan', 'people.person.languages', 'English Language'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Panic! at the Disco'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('m.0n58kgb', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Vanessa Hudgens', 'music.artist.genre', 'Pop music'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot instrumental)'), ('Verse Simmonds', 'people.person.nationality', 'United States of America'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Pop music'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('m.0njw257', 'award.award_honor.award_winner', 'Justin Bieber'), ('Believe', 'common.topic.image', 'justin bieber'), ('m.0zbg2kw', 'award.award_nomination.award', 'American Music Award for Favorite Pop/Rock Album'), ('m.0_vmmj6', 'award.award_nomination.nominated_for', 'Beauty And A Beat'), ('m.0n4rmg7', 'award.award_honor.ceremony', '2011 MTV Video Music Awards'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love (J Stax remix)'), ('m.0gbm3b7', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Beautiful', 'common.topic.notable_types', 'Musical Recording'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Weezer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Iggy Azalea'), ('1Club.FM: Mix 106', 'common.topic.image', '1clubfm.jpg'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ginuwine', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftp1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ywvh8k'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Michael Jackson', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Anastacia', 'music.artist.genre', 'Dance music'), ('Confident', 'music.composition.composer', 'Justin Bieber'), ('Nasri', 'people.person.nationality', 'Canada'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0j8z6tl'), ('m.0101fs_z', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Donna Summer', 'broadcast.artist.content', 'radioIO Classic RNB'), ('Usher', 'people.person.profession', 'Singer-songwriter'), ('Teen idol', 'base.icons.icon_genre.icons', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Zac Hanson'), ('Ja Rule', 'broadcast.artist.content', '.977 The Hits Channel'), ('Sir Nolan', 'music.composer.compositions', 'All Around The World'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Singer', 'people.profession.corresponding_type', 'Musical Artist'), ('Sia Furler', 'common.topic.notable_types', 'Musical Artist'), ('The Notorious B.I.G.', 'people.person.profession', 'Musician'), ('Hikaru Utada', 'music.artist.genre', 'Dance music'), ('Jordan Francis', 'music.artist.genre', 'Synthpop'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fergie'), ('m.0gj0bj6', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Donna Summer', 'music.artist.genre', 'Disco'), ('Savan Kotecha', 'music.artist.genre', 'Contemporary R&B'), ('Pray', 'common.topic.notable_for', 'g.1yp3d4v1g'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Bad Day', 'music.recording.song', 'Bad Day'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c918'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Trance music'), ('Rock music', 'common.topic.subject_of', 'Steve Blevins'), ('#thatPower', 'music.composition.composer', 'Damien LeRoy'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'ABBA'), ('Artist', 'people.profession.specializations', 'Musician'), ('Record producer', 'common.topic.webpage', 'm.09xh0kp'), ('Hot Wired Radio', 'broadcast.content.artist', 'Pearl Jam'), ('Justin Bieber', 'music.featured_artist.albums', 'Live My Life'), ('One Less Lonely Girl', 'common.topic.notable_for', 'g.126sh6292'), ('As Long as You Love Me', 'music.album.album_content_type', 'Remix album'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bigger', 'common.topic.notable_types', 'Composition'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5tj13'), ('#thatPower', 'common.topic.notable_types', 'Composition'), ('.977 The Hits Channel', 'broadcast.content.artist', 'DMX'), ('m.0vp7m_x', 'music.recording_contribution.album', 'Runaway Love (remix)'), ('Die in Your Arms', 'music.composition.composer', 'Justin Bieber'), ('JoJo', 'broadcast.artist.content', '.977 The Hits Channel'), ('PowerHitz', 'broadcast.content.artist', 'Nelly'), ('m.0z85n_1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Baby', 'common.topic.notable_types', 'Musical Recording'), ('Hot Wired Radio', 'broadcast.content.artist', 'Matchbox Twenty'), ('m.0yrlqp1', 'education.education.institution', 'Stratford Northwestern Secondary School'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('My Worlds: The Collection', 'music.album.genre', 'Contemporary R&B'), ('Music executive', 'common.topic.notable_types', 'Profession'), ('Person', 'type.type.properties', 'Profession'), ('FLOW 103', 'broadcast.content.artist', 'Usher'), ('All Around the World', 'music.album.contributor', 'm.0vp800w'), ('Spanish Language', 'language.human_language.countries_spoken_in', 'Canada'), ('Ginuwine', 'people.person.profession', 'Singer-songwriter'), ('Sean Combs', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Sir Mix-a-Lot', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Justin Bieber: Never Say Never', 'film.film.estimated_budget', 'm.0h17ns7'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Pussycat Dolls'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Classic hits'), ('We Are the World 25 for Haiti', 'music.album.album_content_type', 'Studio album'), ('Nelly', 'people.person.profession', 'Actor'), ('50 Cent', 'people.person.gender', 'Male'), ('Wait for a Minute', 'music.composition.composer', 'Jess Jackson'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvm4'), ('Sean Combs', 'people.person.gender', 'Male'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'common.topic.article', 'm.0pc5x25'), ('Michael Jackson', 'influence.influence_node.influenced', 'Usher'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Carl B'), ('Terius Nash', 'broadcast.artist.content', 'PowerHitz'), ('Selena Gomez', 'people.person.gender', 'Female'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Record producer', 'base.ontologies.ontology_instance.equivalent_instances', 'm.09klxxt'), ('Shaffer Smith', 'broadcast.artist.content', 'Sunshine Radio'), ('JellyRadio.com', 'broadcast.content.artist', 'Dr. Dre'), ('m.0n4rmg7', 'award.award_honor.award', 'MTV Video Music Award for Best Male Video'), ('Canadian', 'base.firsts.first_achievement_category.firsts', 'm.0j1t2xs'), ('Baby', 'award.award_winning_work.awards_won', 'm.0njhyh_'), ('1Club.FM: Power', 'common.topic.image', '1clubfm.jpg'), ('Juvenile', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Tyga', 'people.person.gender', 'Male'), ('Sir Nolan', 'people.person.profession', 'Record producer'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Mario'), ('Lil Wayne', 'music.artist.genre', 'Rock music'), ('Lupe Fiasco', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'common.topic.subjects', 'Singer'), ('My Worlds: The Collection', 'music.album.genre', 'Dance-pop'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All Bad', 'music.composition.recordings', 'All Bad'), ('Jessica Simpson', 'people.person.nationality', 'United States of America'), ('Singer', 'people.profession.specializations', 'Jazz Artist/Singer'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Wait For a Minute', 'music.recording.featured_artists', 'Justin Bieber'), ('Hold Tight', 'common.topic.notable_types', 'Composition'), ('LeAnn Rimes', 'music.artist.genre', 'Pop music'), ('m.0v1d2xz', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0k05qcr', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gxnp02', 'base.popstra.dated.participant', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', '50 Cent'), ('HitzRadio.com', 'broadcast.content.artist', 'Pras'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sean Kingston'), ('1Club.FM: Power', 'broadcast.content.location', 'Chicago'), ('R. Kelly', 'people.person.profession', 'Actor'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Pop music'), ('m.0101gx29', 'people.marriage.spouse', 'Jeremy Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'The All-American Rejects'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Contemporary R&B'), ('Winning work', 'rdf-schema#range', 'Award-Winning Work'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Musician'), ('U Smile', 'music.album.release_type', 'Single'), ('Kevin Risto', 'music.composer.compositions', 'Bigger'), ('Juvenile', 'broadcast.artist.content', 'PowerHitz'), ('Dapo Torimiro', 'people.person.profession', 'Record producer'), ('Enrique Iglesias', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvfs'), ('Runaway Love (remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.album.releases', 'As Long as You Love Me'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Reggae', 'music.genre.parent_genre', 'World music'), ('Benny Blanco', 'people.person.gender', 'Male'), ('All That Matters', 'music.composition.composer', 'Dre & Vidal'), ('Brandy Norwood', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Rihanna', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Next to You', 'music.album.featured_artists', 'Justin Bieber'), ('m.0101fv74', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jessica Jarrell'), ('RBMG Records', 'common.topic.article', 'm.0gmfc7y'), ('P!nk', 'people.person.profession', 'Model'), ('Usher', 'music.artist.genre', 'Dance-pop'), ('Jeremy Bieber', 'people.person.place_of_birth', 'Stratford'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_1wm'), ('Lil Jon', 'people.person.languages', 'English Language'), ('Indie rock', 'music.genre.parent_genre', 'Rock music'), ('1.FM Top 40', 'broadcast.content.artist', 'Goldfrapp'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Rockumentary'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Never Say Never', 'music.album.artist', 'Justin Bieber'), ('m.0ncpx3v', 'organization.leadership.person', 'Barry Weiss'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Britney Spears'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Profession'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvp7'), ('Sean Combs', 'people.person.profession', 'Actor'), ('Barry Weiss', 'people.person.nationality', 'United States of America'), ('Synthpop', 'music.genre.parent_genre', 'Synthpop'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'people.person.sibling_s', 'm.0gxnnwp'), ('Thought Of You', 'music.composition.recordings', 'Thought of You'), ('Akon', 'broadcast.artist.content', 'FLOW 103'), ('Toby Gad', 'people.person.profession', 'Musician'), ('Rob Thomas', 'people.person.profession', 'Singer'), ('Teen Choice Award for Choice Music: Album Pop', 'award.award_category.winners', 'm.0z8qx3w'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.010lkp2z', 'award.award_honor.award_winner', 'Justin Bieber'), ('Gavin DeGraw', 'people.person.profession', 'Musician'), ('Reed Smoot', 'freebase.valuenotation.has_value', 'Parents'), ('Katy Perry: Part of Me', 'film.film.directed_by', 'Dan Cutforth'), ('Yves Bole', 'freebase.valuenotation.has_no_value', 'Notable types'), ('Enrique Iglesias', 'people.person.profession', 'Model'), ('Donna Summer', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Shaun Cassidy', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0rqh7d9'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('PowerHitz', 'broadcast.content.artist', 'Rihanna'), ('m.0njhtjj', 'award.award_honor.award_winner', 'Justin Bieber'), ('Terence Dudley', 'music.artist.genre', 'Rhythm and blues'), ('Lady Gaga', 'people.person.profession', 'Film Producer'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('m.0z3vx9q', 'music.music_video_performance.special_music_video_performance_type', 'Dancer'), ('All That Matters', 'music.album.artist', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Dapo Torimiro', 'people.person.gender', 'Male'), ('Musical Album', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beautiful', 'music.recording.song', 'Beautiful'), ('Never Let You Go', 'music.composition.lyricist', 'Bryan-Michael Cox'), ('m.043082k', 'common.webpage.topic', '1Club.FM: Channel One'), ('PowerHitz', 'broadcast.content.genre', 'Pop music'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('JellyRadio.com', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntw'), ('P!nk', 'people.person.profession', 'Singer-songwriter'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Urban contemporary', 'broadcast.genre.content', 'WildFMRadio.com'), ('m.0jvgmxc', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j21b47'), ('Ciara', 'broadcast.artist.content', 'WildFMRadio.com'), ('Amerie', 'people.person.profession', 'Record producer'), ('Urban contemporary', 'broadcast.genre.content', 'Emphatic Radio.com!'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Award category'), ('1.FM Top 40', 'broadcast.content.artist', 'Maroon 5'), ('Will i Am', 'people.person.profession', 'Singer'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.0101fsz2', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Justin Bieber: Never Say Never', 'film.film.costume_design_by', 'Kurt Swanson'), ('Alicia Keys', 'broadcast.artist.content', '1Club.FM: V101'), ('Beyoncé Knowles', 'broadcast.artist.content', 'FLOW 103'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me'), ('Beauty And A Beat', 'music.composition.language', 'English Language'), ('Change Me', 'music.album.release_type', 'Single'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Usher'), ('Sean Kingston', 'music.artist.origin', 'Miami'), ('Teen idol', 'common.topic.webpage', 'm.09xx941'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flo Rida'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Under the Mistletoe', 'award.award_winning_work.awards_won', 'm.0z8755b'), ('Yves Bole', 'freebase.valuenotation.has_no_value', 'Image'), ('Selena Gomez', 'base.popstra.celebrity.dated', 'm.0gxnp02'), ('Boyfriend', 'music.single.versions', 'Boyfriend (acoustic version)'), ('Jordan Pruitt', 'people.person.nationality', 'United States of America'), ('Lady Gaga', 'music.artist.genre', 'Dance-pop'), ('Runaway Love (remix)', 'common.topic.notable_types', 'Musical Album'), ('Beauty and a Beat (Remixes)', 'music.album.primary_release', 'Beauty and a Beat (Remixes)'), ('Hot Wired Radio', 'broadcast.content.artist', 'Red Hot Chili Peppers'), ('Rihanna', 'people.person.profession', 'Actor'), ('Mary J. Blige', 'people.person.profession', 'Singer-songwriter'), ('Kylie Minogue', 'broadcast.artist.content', '1.FM Top 40'), ('Ice Cube', 'people.person.nationality', 'United States of America'), ('Musical Album', 'type.type.properties', 'Artist'), ('Blu Cantrell', 'music.artist.genre', 'Rhythm and blues'), ('Beauty and a Beat (Wideboys Dub)', 'common.topic.notable_types', 'Musical Recording'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Montell Jordan', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'K-Ci & JoJo'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO RNB Mix'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chingy'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The All-American Rejects'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('iJustine', 'influence.influence_node.influenced_by', 'Yves Bole'), ('m.0_cyzs_', 'celebrities.legal_entanglement.location', 'Miami'), ('Judy Garland', 'people.person.languages', 'English Language'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sia Furler', 'people.person.profession', 'Singer'), ('m.0101ft35', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('m.0v_729v', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Shaffer Smith', 'people.person.gender', 'Male'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Rhythm and blues'), ('Terence Dudley', 'people.person.nationality', 'United States of America'), ('m.0gbm3g2', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Singer', 'people.profession.specializations', 'Opera Singer'), ('m.012bq275', 'celebrities.friendship.friend', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gttv'), ('Boyfriend', 'music.album.releases', 'Boyfriend'), ('Deborah Lurie', 'film.music_contributor.film', 'Justin Bieber: Never Say Never'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0zg2w8r', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('m.0f0dwc4', 'common.webpage.category', 'Curated Topic'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5m'), ('JoJo', 'common.topic.notable_types', 'Musical Artist'), ('m.046c33q', 'common.webpage.topic', 'Emphatic Radio.com!'), ('MTV Movie Award for Best Jaw Dropping Moment', 'award.award_category.nominees', 'm.0pc67ly'), ('Eenie Meenie', 'music.composition.composer', 'Benny Blanco'), ('m.0y_g42w', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Journals', 'common.topic.notable_types', 'Musical Album'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPower'), ('Believe', 'music.album.genre', 'Hip hop music'), ('Musician', 'people.profession.specialization_of', 'Artist'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2v'), ('Jaden Smith', 'people.person.nationality', 'United States of America'), ('Bigger', 'music.composition.lyricist', 'Waynne Nugent'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('JellyRadio.com', 'broadcast.content.genre', 'Top 40'), ('m.0z3vx9q', 'music.music_video_performance.special_music_video_performance_type', 'Actor'), ('All Around the World', 'music.single.versions', 'All Around The World (featuring Ludacris)'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Estelle', 'music.artist.genre', 'Hip hop music'), ('JoJo', 'music.artist.genre', 'Contemporary R&B'), ('Leonardo DiCaprio', 'people.person.profession', 'Actor'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love (J Stax remix)'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4r4'), ('Kuk Harrell', 'music.artist.genre', 'Pop music'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0y_g42w'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Khalil'), ('Yves Bole', 'people.person.gender', 'Male'), ('Runaway Love (remix)', 'music.album.featured_artists', 'Raekwon'), ('m.0ndc259', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z3tqqt'), ('m.0sxhq3d', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('PowerHitz', 'broadcast.content.artist', 'Baby Bash'), ('Elvis Presley', 'music.artist.genre', 'Rhythm and blues'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0n58kgb', 'award.award_nomination.nominated_for', 'Believe'), ('m.0pbzq13', 'film.performance.actor', 'Justin Bieber'), ('Yves Bole', 'music.artist.genre', 'Pop music'), ('Sunshine Radio', 'common.topic.article', 'm.03hr_cl'), ('Raekwon', 'people.person.nationality', 'United States of America'), ('Chloe Bridges', 'people.person.gender', 'Female'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0z1jn32'), ('Will i Am', 'people.person.profession', 'Actor'), ('Rita Ora', 'freebase.valuenotation.has_no_value', 'Children'), ('m.09klxxt', 'base.ontologies.ontology_instance_mapping.freebase_topic', 'Record producer'), ('Fabolous', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnvf'), ('Britney Spears', 'people.person.profession', 'Actor'), ('As Long As You Love Me (Ferry Corsten remix)', 'common.topic.notable_types', 'Musical Recording'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Michael Jackson', 'music.artist.genre', 'Dance music'), ('Journals', 'music.album.album_content_type', 'Compilation album'), ('Somebody to Love', 'music.recording.featured_artists', 'Usher'), ('Akon', 'music.artist.genre', 'Dance music'), ('Adam Messinger', 'music.artist.genre', 'Pop music'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvj9'), ('Yves Bole', 'celebrities.celebrity.celebrity_rivals', 'm.012bm5cg'), ('Alanis Morissette', 'music.artist.genre', 'Dance-pop'), ('Boyfriend', 'music.composition.language', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'David Nicksay'), ('Justin Timberlake', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Rodney Jerkins', 'music.artist.genre', 'Hip hop music'), ('Twista', 'people.person.place_of_birth', 'Chicago'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Rock music'), ('Benny Blanco', 'people.person.profession', 'Record producer'), ('Mary J. Blige', 'broadcast.artist.content', 'Hot 108 Jamz'), ('All Around the World', 'music.recording.artist', 'Ludacris'), ('Right Here', 'music.composition.composer', 'Hit-Boy'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Tristan Prettyman'), ('Diplo', 'music.featured_artist.recordings', 'Where Are Ü Now'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3fj'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjql'), ('L.A. Reid', 'music.artist.genre', 'Contemporary R&B'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg2qjr'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.artist', 'Justin Bieber'), ('m.0sgkrj4', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('Justin Bieber: Never Say Never', 'film.film.country', 'United States of America'), ('R. Kelly', 'people.person.profession', 'Artist'), ('m.0101fvfs', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Lolly', 'common.topic.notable_for', 'g.11b75r5d3z'), ('1.FM Top 40', 'broadcast.content.artist', 'Madonna'), ('m.0k05qcr', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('DMX', 'people.person.gender', 'Male'), ('Colbie Caillat', 'people.person.gender', 'Female'), ('m.0j_tkcq', 'film.personal_film_appearance.film', \"Justin Bieber: Never Say Never - Director's Fan Cut\"), ('Believe', 'music.album.releases', 'Believe'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulClassics.com'), ('Ludacris', 'broadcast.artist.content', 'WildFMRadio.com'), ('Enrique Iglesias', 'people.person.profession', 'Record producer'), ('1Club.FM: Mix 106', 'common.topic.notable_for', 'g.1255j19v0'), ('Nick Jonas', 'music.artist.genre', 'Dance music'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Shaffer Smith'), ('My Worlds: The Collection', 'music.album.genre', 'Pop music'), ('Baby', 'music.composition.recordings', 'Baby'), ('JellyRadio.com', 'broadcast.content.artist', 'Sean Paul'), ('Beyoncé Knowles', 'music.artist.genre', 'Teen pop'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), (\"Justin Bieber's Believe\", 'film.film.directed_by', 'Jon M. Chu'), ('Akon', 'music.artist.genre', 'Pop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Hold Tight', 'music.composition.composer', 'Dominic Jordan'), ('1.FM Top 40', 'broadcast.content.artist', 'Leona Lewis'), ('Snoop Dogg', 'people.person.profession', 'Musician'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Duffy', 'music.artist.genre', 'Dance-pop'), ('m.0d_hbgr', 'common.webpage.topic', 'Justin Bieber'), ('Britney Spears', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0pcr2dh', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never'), ('Rodney Jerkins', 'music.artist.genre', 'Rock music'), ('m.0t4s_bn', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Adrienne Bailon', 'people.person.nationality', 'United States of America'), ('Image', 'rdf-schema#domain', 'Topic'), ('Nicki Minaj', 'people.person.profession', 'Singer'), ('m.0z83x9l', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.012bm2cw', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Will Smith', 'music.artist.genre', 'Pop music'), ('Shaffer Smith', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Charlie Cohen'), ('Love Never Felt So Good', 'music.composition.composer', 'Michael Jackson'), ('Contemporary R&B', 'broadcast.genre.content', 'Classic Soul Network'), ('Jay Cassidy', 'film.editor.film', 'Justin Bieber: Never Say Never'), ('Blu Cantrell', 'people.person.profession', 'Singer-songwriter'), ('Judy Garland', 'base.icons.icon.icon_genre', 'Teen idol'), ('Mariah Carey', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Dan Cutforth', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Enrique Iglesias', 'music.artist.genre', 'Synthpop'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Guglielmo Scilla'), ('m.0z84kwx', 'award.award_nomination.nominated_for', 'Boyfriend'), ('Nelly Furtado', 'people.person.profession', 'Record producer'), ('Aaliyah', 'broadcast.artist.content', 'HitzRadio.com'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Jay-Z', 'people.person.profession', 'Singer'), ('DMX', 'broadcast.artist.content', 'Smoothbeats'), ('SoulfulHipHop.com Radio', 'common.topic.notable_types', 'Broadcast Content'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Sean Kingston'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Erasure'), ('Justin Bieber', 'music.artist.genre', 'Pop music'), ('Kylie Minogue', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Nelly', 'music.artist.genre', 'Pop music'), ('Right Here', 'common.topic.notable_for', 'g.1yl5x2jhc'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_types', 'Composition'), ('Emphatic Radio.com!', 'broadcast.content.artist', '50 Cent'), ('Island Records', 'organization.organization.parent', 'm.05sp405'), ('Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9h8f'), ('Justin Bieber', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0v_714c'), ('Rita Ora', 'people.person.gender', 'Female'), ('Yves Bole', 'music.artist.album', 'You'), ('Drake', 'people.person.profession', 'Actor'), ('Blackstreet', 'music.artist.genre', 'Electronic dance music'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.song', 'As Long as You Love Me'), ('181-beat', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('Rudolph Isley', 'people.person.profession', 'Singer'), ('m.0_syttc', 'award.award_nomination.nominated_for', 'justinbieber'), ('Live My Life', 'music.composition.composer', 'Kev Nish'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Blank & Jones'), ('Never Say Never', 'music.composition.recorded_as_album', 'Never Say Never'), ('Anastacia', 'people.person.gender', 'Female'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0v90p2b'), ('Whitney Houston', 'influence.influence_node.influenced', 'Beyoncé Knowles'), ('Aaliyah', 'people.person.profession', 'Model'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.010htdc2'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('Demi Lovato', 'music.artist.label', 'Island Records'), ('Teen pop', 'common.topic.webpage', 'm.09wmk1t'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award category'), ('All That Matters', 'music.single.versions', 'All That Matters (video)'), ('All Around The World', 'common.topic.notable_for', 'g.125829xyd'), ('Pattie Mallette', 'common.topic.article', 'm.0h5n_50'), ('Justin Bieber', 'people.person.education', 'm.0w5l5h1'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Katy Perry'), ('Nick Jonas', 'music.artist.genre', 'Teen pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvs9s'), ('Country of nationality', 'rdf-schema#range', 'Country'), ('m.0101fvps', 'film.film_regional_release_date.film_release_region', 'Italy'), ('Drake', 'music.artist.genre', 'Pop music'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Record producer', 'common.topic.subject_of', 'Alan Motley'), ('Rihanna', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'music.album.release_type', 'Single'), ('New Kids on the Block', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('1Club.FM: Channel One', 'common.topic.notable_for', 'g.1257kpjrj'), ('radioIO Todays RNB', 'broadcast.content.artist', \"Destiny's Child\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Three Days Grace'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5ld8'), ('m.0gbm3fj', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Britney Spears', 'broadcast.artist.content', '1.FM Top 40'), ('Brandy Norwood', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Dany Brillant', 'music.artist.genre', 'Pop music'), ('m.0y5t8gm', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhmll'), ('Lolly', 'music.recording.song', 'Lolly'), ('CSI: Crime Scene Investigation', 'tv.tv_program.languages', 'English Language'), ('1Club.FM: Power', 'broadcast.content.artist', '112'), ('Dr. Dre', 'people.person.profession', 'Artist'), ('m.0pc67ly', 'award.award_nomination.nominated_for', 'Justin Bieber: Never Say Never'), ('m.0vpgdpb', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Jaxon Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.genre', \"00's\"), ('Kylie Minogue', 'music.artist.genre', 'Eurodance'), ('Rodney Jerkins', 'people.person.profession', 'Singer'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.featured_artists', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mario'), ('Right Here', 'music.composition.recordings', 'Right Here (featuring Drake)'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Contemporary R&B', 'music.music_video_genre.music_videos_of_this_genre', 'Josue Diaz'), ('1Club.FM: V101', 'common.topic.image', '1clubfm.jpg'), (\"Justin Bieber's Believe\", 'film.film.genre', 'Documentary film'), ('Nick Jonas', 'people.person.profession', 'Singer'), ('Jennifer Lopez', 'music.artist.genre', 'Dance-pop'), ('m.0101ft0d', 'film.personal_film_appearance.person', 'Mike Posner'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Rita Ora', 'music.artist.genre', 'Pop music'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_721l'), ('m.0ywtfj6', 'award.award_nomination.ceremony', '5th Annual Shorty Awards'), ('Hikaru Utada', 'music.artist.label', 'The Island Def Jam Music Group'), ('Linkin Park', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Green Day'), ('Pattie Mallette', 'people.person.parents', 'Diane Mallette'), ('Everlast', 'common.topic.notable_types', 'Musical Artist'), ('m.0gbmnsr', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('My World', 'music.album.genre', 'Contemporary R&B'), ('My World', 'music.album.releases', 'My World'), ('Tampa', 'common.topic.notable_types', 'City/Town/Village'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Vanessa Hudgens', 'people.person.nationality', 'United States of America'), (\"Kids' Choice Award for Biggest Slime Of The Night\", 'award.award_category.winners', 'm.0sgkw_d'), ('Wait for a Minute', 'common.topic.notable_types', 'Composition'), ('#thatPOWER', 'music.album.release_type', 'Single'), ('Lolly', 'common.topic.notable_types', 'Musical Album'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bl_p'), ('Justin Bieber', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0101ftmz', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('m.0gbm3cp', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Justin Bieber', 'base.popstra.celebrity.dated', 'm.0gxnp0c'), ('Kevin Risto', 'freebase.valuenotation.has_value', 'Date of birth'), ('Eminem', 'music.artist.genre', 'Hip hop music'), ('NME Award for Villain of the Year', 'award.award_category.winners', 'm.0z83x9l'), ('Nicki Minaj', 'music.composer.compositions', 'Beauty And A Beat'), ('Hot Wired Radio', 'broadcast.content.artist', 'Boys Like Girls'), ('Kylie Minogue', 'music.artist.genre', 'Disco'), ('Akon', 'broadcast.artist.content', 'PowerHitz'), ('Amerie', 'music.artist.genre', 'Hip hop music'), ('FLOW 103', 'broadcast.content.artist', 'Beyoncé Knowles'), (\"Justin Bieber's Believe\", 'film.film.costume_design_by', 'Kiyomi Hara'), ('Jay-Z', 'people.person.nationality', 'United States of America'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Spouse (or domestic partner)', 'rdf-schema#domain', 'Person'), ('m.0zg2w8r', 'award.award_nomination.nominated_for', 'My World 2.0'), (\"Justin Bieber's Believe\", 'film.film.prequel', 'Justin Bieber: Never Say Never'), ('Justin Bieber', 'people.person.place_of_birth', 'London'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Everlast', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.producer', 'Beirut Nights Radio'), ('P!nk', 'music.artist.genre', 'Contemporary R&B'), ('Hip hop music', 'broadcast.genre.content', '1Club.FM: Power'), ('Justin Bieber: Just Getting Started', 'common.topic.notable_for', 'g.1259prs0m'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('PowerHitz', 'broadcast.content.artist', 'Ludacris'), ('Zac Efron', 'people.person.profession', 'Actor'), ('m.0101fv97', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('L.A. Reid', 'people.person.profession', 'Record producer'), ('m.0zg2qjr', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Kelis', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.featured_artist.recordings', 'Wont Stop (feat. Justin Bieber)'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Linkin Park'), ('Tupac Shakur', 'broadcast.artist.content', '.977 The Hits Channel'), ('Fall Out Boy', 'broadcast.artist.content', 'Hot Wired Radio'), ('Lady Gaga', 'people.person.languages', 'English Language'), ('Deborah Lurie', 'music.artist.genre', 'Pop music'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkgd6'), ('Estelle', 'music.artist.genre', 'Reggae'), ('Wait For a Minute', 'common.topic.notable_types', 'Musical Album'), ('Real Change: Artists for Education', 'common.topic.notable_types', 'Film'), ('Aaliyah', 'music.artist.genre', 'Contemporary R&B'), ('m.0bvmhvb', 'common.webpage.topic', 'Justin Bieber'), ('A Star Was Born: Justin Bieber', 'media_common.netflix_title.netflix_genres', 'Documentary film'), ('Music Producer', 'people.profession.specializations', 'Record producer'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Kelly Clarkson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.0y7y53r', 'award.award_honor.award', 'Kerrang! Villain of the Year Award'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chrisette Michele'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fall Out Boy'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vnr'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y4tdml'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Paula DeAnda'), ('Next to You', 'music.single.versions', 'Next to You'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('Kelis', 'music.artist.genre', 'Rock music'), ('m.0zbf_4g', 'freebase.valuenotation.has_no_value', 'Winning work'), ('2014 Billboard Music Awards', 'time.event.people_involved', 'Ludacris'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0gj1yzm', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0h4yhbb', 'film.personal_film_appearance.film', 'A Star Was Born: Justin Bieber'), ('First Dance', 'music.composition.lyricist', 'Justin Bieber'), ('Kanye West', 'broadcast.artist.content', '1Club.FM: Power'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cw'), ('Contemporary R&B', 'broadcast.genre.content', 'PowerHitz'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Santana', 'broadcast.artist.content', '1Club.FM: V101'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv4k'), ('m.0101fs_z', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Giusy Ferreri'), ('Soulja Boy', 'music.artist.genre', 'Pop music'), ('Nick Jonas', 'people.person.profession', 'Record producer'), ('Believe', 'music.album.album_content_type', 'Studio album'), ('Confident', 'music.recording.artist', 'Justin Bieber'), ('Ginuwine', 'people.person.profession', 'Actor'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0yrjkl1', 'award.award_honor.award_winner', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('The Isley Brothers', 'broadcast.artist.content', 'Classic Soul Network'), ('m.0pbzq13', 'film.performance.special_performance_type', 'Uncredited'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Somebody To Love', 'common.topic.notable_types', 'Musical Recording'), ('m.0115qhzk', 'award.award_honor.ceremony', '2011 MTV Europe Music Awards'), ('Hot Wired Radio', 'broadcast.content.artist', 'Justin Bieber'), ('Book', 'freebase.type_hints.included_types', 'Topic'), ('Pattie Mallette', 'people.person.place_of_birth', 'Stratford'), ('Yves Bole', 'people.person.education', 'm.0129j_53'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Official website'), ('First Dance', 'common.topic.image', 'Usher OMG Houston'), ('August Rigo', 'people.person.profession', 'Record producer'), ('Jeremy Bieber', 'people.person.spouse_s', 'm.0101gx29'), ('Kelly Clarkson', 'music.artist.genre', 'Rhythm and blues'), ('All Around The World (featuring Ludacris)', 'music.recording.song', 'All Around The World'), ('Canada', 'base.aareas.schema.administrative_area.administrative_children', 'Ontario'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Pray', 'common.topic.article', 'm.0g5b0p6'), ('Turn to You (Mother’s Day Dedication)', 'music.album.release_type', 'Single'), ('Akon', 'freebase.valuenotation.has_value', 'Children'), ('Die in Your Arms', 'music.album.artist', 'Justin Bieber'), ('Chance the Rapper', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.genre', 'Urban contemporary'), ('m.0y_g556', 'award.award_honor.honored_for', 'justinbieber'), ('m.0pcr28j', 'tv.tv_guest_role.episodes_appeared_in', 'Targets of Obsession'), ('radioIO Todays POP', 'common.topic.notable_types', 'Broadcast Content'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Beyoncé Knowles'), ('m.0yr9tjb', 'award.award_nomination.award', 'Young Hollywood Award for Newcomer of the Year'), ('Nicki Minaj', 'music.artist.genre', 'Synthpop'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.nominees', 'm.0z84kwx'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Cris Cab', 'people.person.profession', 'Singer-songwriter'), ('Aaliyah', 'people.person.nationality', 'United States of America'), ('K-Ci & JoJo', 'music.artist.genre', 'Rhythm and blues'), ('Maroon 5', 'music.artist.genre', 'Pop music'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miley Cyrus', 'people.person.languages', 'English Language'), ('Beauty and a Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0z87597'), ('m.0pcr2dh', 'tv.tv_guest_role.episodes_appeared_in', 'Shock Waves'), ('Eminem', 'people.person.profession', 'Musician'), ('m.0z8qx3w', 'award.award_honor.honored_for', 'My World 2.0'), ('1Club.FM: Power', 'broadcast.content.genre', 'Top 40'), ('Justin Bieber', 'common.topic.webpage', 'm.0gc_9w6'), ('Lady Antebellum', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Miley Cyrus', 'people.person.profession', 'Singer-songwriter'), ('Tupac Shakur', 'people.person.profession', 'Record producer'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juvenile', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Will i Am', 'people.person.nationality', 'United States of America'), ('Terence Dudley', 'music.artist.label', 'The Island Def Jam Music Group'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Terius Nash'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Music & Concert Documentaries'), ('Chris Brown', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('#Thatpower', 'music.recording.artist', 'Will i Am'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Rock music'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Scooter Braun', 'music.artist.label', 'RBMG Records'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Britney Spears', 'music.artist.genre', 'Dance-pop'), ('Soulja Boy', 'broadcast.artist.content', 'WildFMRadio.com'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Sean Combs', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'music.composer.compositions', 'PYD'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Big R Radio - Top 40 Hits', 'common.topic.article', 'm.03wyslv'), ('m.0w3gbtv', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Shaffer Smith'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Blu Cantrell'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mims'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bryan Adams', 'people.person.profession', 'Musician'), ('Ashlee Simpson', 'people.person.nationality', 'United States of America'), ('Hold Tight', 'common.topic.notable_types', 'Musical Recording'), ('Bad Day', 'common.topic.notable_for', 'g.1yl5wl7k0'), ('Sunshine Radio', 'broadcast.content.location', 'Nyíregyháza'), ('Person', 'freebase.type_profile.strict_included_types', 'Topic'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ernie Isley', 'music.artist.genre', 'Rhythm and blues'), ('MTV Video Music Award for Best Male Video', 'award.award_category.winners', 'm.0n4rmg7'), ('Beyoncé Knowles', 'music.artist.genre', 'Contemporary R&B'), ('Jason Mraz', 'people.person.profession', 'Actor'), ('Miami', 'location.location.containedby', 'United States of America'), ('Justin Bieber: Just Getting Started', 'book.book.genre', 'Autobiography'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.artist', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Sean Paul'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Madonna'), ('Lupe Fiasco', 'people.person.profession', 'Musician'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Third Eye Blind'), ('Anastacia', 'music.artist.genre', 'Rhythm and blues'), ('Musician', 'people.profession.corresponding_type', 'Musical Artist'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0tjyljn'), ('m.0v_714c', 'film.personal_film_appearance.film', 'Disney Parks Christmas Day Parade'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Khalil', 'people.person.profession', 'Singer'), ('Selena Gomez', 'base.popstra.business_location.customer', 'm.0gxnp5d'), ('Never Say Never (acoustic)', 'music.recording.song', 'Never Say Never'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('The Pussycat Dolls', 'broadcast.artist.content', 'radioIO Todays POP'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Height'), ('Katy Perry: Part of Me', 'film.film.produced_by', 'Katy Perry'), ('m.0gbm3bs', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Tommy Sands', 'base.icons.icon.icon_genre', 'Teen idol'), ('Khalil', 'people.person.nationality', 'United States of America'), ('1.FM Top 40', 'broadcast.content.artist', 'Rihanna'), ('Mary J. Blige', 'people.person.profession', 'Actor'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Frankie J'), ('Beyoncé Knowles', 'people.person.gender', 'Female'), ('Gwen Stefani', 'broadcast.artist.content', 'DeeGay'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.012f2m86', 'film.film_film_distributor_relationship.film', \"Justin Bieber's Believe\"), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k4t'), ('All Around The World', 'music.composition.language', 'English Language'), ('m.0z87597', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('m.0101ftnq', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Blu Cantrell', 'music.artist.genre', 'Pop music'), ('m.0ndc3_1', 'award.award_honor.award', 'American Music Award for Artist of the Year'), ('All Around The World', 'music.composition.composer', 'Sir Nolan'), ('Heartbreaker', 'music.composition.recorded_as_album', 'Heartbreaker'), ('Ciara', 'people.person.languages', 'English Language'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvf2'), ('R. Kelly', 'people.person.profession', 'Record producer'), ('Hold Tight', 'common.topic.notable_for', 'g.1yl5pzyrs'), ('Shaffer Smith', 'music.artist.genre', 'Contemporary R&B'), ('Heartbreaker', 'music.album.genre', 'Rhythm and blues'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mick Quinn'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Pop music'), ('Shaffer Smith', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gd28hv'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Usher', 'film.person_or_entity_appearing_in_film.films', 'm.0101fszb'), ('Janet Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Rock music'), ('The Isley Brothers', 'music.artist.genre', 'Rock music'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mario'), ('Believe Acoustic', 'music.album.genre', 'Acoustic music'), ('SoulfulSmoothJazz.com', 'common.topic.notable_types', 'Broadcast Content'), ('PowerHitz', 'broadcast.content.artist', 'Beyoncé Knowles'), ('.977 The Hits Channel', 'broadcast.content.artist', \"Plain White T's\"), ('Somebody to Love', 'common.topic.notable_types', 'Musical Album'), ('Electronic dance music', 'music.genre.subgenre', 'Synthpop'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Coldplay'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.episode', 'Series 7 - Elimination - 8'), ('The Pussycat Dolls', 'music.artist.genre', 'Dance music'), ('Thought of You', 'music.single.versions', 'Thought of You'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Trick Daddy'), ('First Dance', 'common.topic.notable_types', 'Composition'), ('Shaffer Smith', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Snoop Dogg', 'music.artist.genre', 'Pop music'), ('John Mamann', 'music.composer.compositions', 'Live My Life'), ('New Kids on the Block', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njgyk4'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Will i Am', 'music.artist.genre', 'Dance music'), ('Enrique Iglesias', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Top 40'), ('m.0rqp4h0', 'music.track_contribution.role', 'Vocals'), ('Beauty And A Beat', 'music.composition.recorded_as_album', 'Beauty and a Beat'), ('Kanye West', 'people.person.profession', 'Actor'), ('m.0101fv62', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('1Club.FM: Power', 'broadcast.content.artist', 'Flo Rida'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftqp'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Jazmyn Bieber', 'common.topic.notable_types', 'Person'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_72tb'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Vanessa Hudgens', 'music.artist.genre', 'Rhythm and blues'), ('Mary J. Blige', 'people.person.nationality', 'United States of America'), ('Jason Mraz', 'people.person.profession', 'Artist'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Black Eyed Peas', 'common.topic.notable_types', 'Musical Artist'), ('Paul Anka', 'people.person.profession', 'Singer'), ('Somebody to Love', 'music.album.artist', 'Justin Bieber'), ('Marvin Isley', 'music.artist.label', 'The Island Def Jam Music Group'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Mariah Carey', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('MTV Europe Music Award for Best Pop', 'award.award_category.winners', 'm.0njw59_'), ('m.0yrhmll', 'award.award_honor.award_winner', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'music.recording.canonical_version', 'Live My Life'), ('David Cassidy', 'people.person.gender', 'Male'), ('Nicki Minaj', 'music.artist.genre', 'Dance music'), ('Lil Jon', 'people.person.profession', 'Record producer'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('Never Say Never', 'music.single.versions', 'Never Say Never'), ('Love Never Felt So Good', 'music.composition.composer', 'Paul Anka'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhtjj'), ('R. Kelly', 'people.person.place_of_birth', 'Chicago'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big Sean', 'people.person.profession', 'Musician'), ('Kelly Clarkson', 'people.person.profession', 'Actor'), (\"2011 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgk_cw'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z2dr9y'), ('Amerie', 'broadcast.artist.content', 'radioIO Todays POP'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO Todays POP'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Bronx'), ('Justin Bieber', 'music.composer.compositions', 'Heartbreaker'), ('Jessie J', 'music.artist.genre', 'Dance music'), ('Cory Gunz', 'people.person.gender', 'Male'), ('Bryan Adams', 'people.person.profession', 'Singer-songwriter'), ('Baby (acoustic)', 'music.recording.song', 'Pray'), ('My Worlds Acoustic', 'music.album.artist', 'Justin Bieber'), ('Kylie Minogue', 'music.artist.genre', 'Synthpop'), ('Gwen Stefani', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.featured_artist.recordings', 'Gas Pedal'), ('Beyoncé Knowles', 'music.artist.genre', 'Urban contemporary'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv8j'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('m.0d33gyj', 'celebrities.romantic_relationship.relationship_type', 'Dated'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftmm'), ('Jennifer Lopez', 'music.artist.genre', 'Hip hop music'), ('m.0yrlqp1', 'freebase.valuenotation.has_value', 'End Date'), ('As Long as You Love Me', 'music.album.featured_artists', 'Big Sean'), ('#thatPOWER', 'music.recording.tracks', 'ThatPower'), ('Jennifer Lopez', 'people.person.nationality', 'United States of America'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Michael Jackson', 'music.artist.genre', 'Dance-pop'), ('Lolly', 'music.album.featured_artists', 'Juicy J'), ('Sunshine Radio', 'common.topic.notable_for', 'g.125dhx24_'), ('Ciara', 'music.artist.genre', 'Pop music'), ('Usher', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Singer-songwriter', 'common.topic.notable_types', 'Profession'), ('First Dance', 'music.composition.form', 'Song'), ('Rodney Jerkins', 'music.composer.compositions', 'Roller Coaster'), ('m.0j8mhs_', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Model', 'common.topic.notable_types', 'Profession'), ('1Club.FM: Power', 'broadcast.content.genre', 'Contemporary R&B'), ('Mannie Fresh', 'music.artist.label', 'The Island Def Jam Music Group'), ('All Bad', 'music.composition.composer', 'Dre & Vidal'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Green Day'), ('m.012m1w0t', 'music.group_membership.role', 'Record producer'), ('Ciara', 'broadcast.artist.content', 'Hot Wired Radio'), ('Linkin Park', 'common.topic.notable_types', 'Musical Artist'), ('Redfoo', 'music.artist.genre', 'Synthpop'), ('m.0j8z6tl', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Maroon 5', 'broadcast.artist.content', 'Hot Wired Radio'), ('Runaway Love', 'music.recording.artist', 'Justin Bieber'), ('Paul Anka', 'music.composer.compositions', 'Love Never Felt So Good'), ('Boyfriend', 'music.composition.composer', 'Mike Posner'), ('Justin Bieber', 'music.artist.contribution', 'm.0vpgdpb'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0zg8bc1', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('m.0ywvh8k', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0t_l7mp', 'award.award_nomination.ceremony', '2013 Billboard Music Awards'), ('Avery', 'people.person.nationality', 'United States of America'), ('Alicia Keys', 'broadcast.artist.content', '1Club.FM: Channel One'), ('FLOW 103', 'broadcast.content.artist', 'Alicia Keys'), ('Bigger', 'common.topic.article', 'm.0fqlv5q'), ('Toby Gad', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5th3r'), ('Snoop Dogg', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0nh04xx'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z0tgz6'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Scott Manson'), ('Justin Bieber', 'music.artist.album', 'Beauty and a Beat (Remixes)'), ('Nick Jonas', 'music.artist.label', 'Island Records'), ('My World 2.0', 'music.album.genre', 'Teen pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjhc6c'), ('#thatPOWER', 'music.recording.tracks', '#ThatPower (feat. Justin Bieber)'), ('Hold Tight', 'music.album.releases', 'Hold Tight'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten remix)'), ('Eminem', 'broadcast.artist.content', '1Club.FM: Power'), ('DMX', 'people.person.profession', 'Film Producer'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Bad Day'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Jessie J'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Victoria Justice', 'people.person.gender', 'Female'), ('Singing', 'people.professional_field.professions_in_this_field', 'Singer-songwriter'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('WildFMRadio.com', 'broadcast.content.artist', 'Timbaland'), ('Jessica Simpson', 'people.person.languages', 'English Language'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Height'), ('Emphatic Radio.com!', 'common.topic.webpage', 'm.046c33q'), ('Alanis Morissette', 'music.artist.genre', 'Pop music'), ('Ja Rule', 'people.person.profession', 'Actor'), ('HitzRadio.com', 'broadcast.content.artist', 'Evanescence'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('A Star Was Born: Justin Bieber', 'common.topic.notable_types', 'Film'), ('Hip hop music', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), ('PYD', 'music.album.featured_artists', 'R. Kelly'), ('Gas Pedal', 'music.recording.artist', 'Justin Bieber'), ('m.011m26pv', 'music.group_membership.group', 'Young Artists for Haiti'), ('Ice Cube', 'common.topic.notable_types', 'Musical Artist'), ('Ciara', 'people.person.profession', 'Singer'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Christina Aguilera'), (\"Destiny's Child\", 'music.artist.genre', 'Urban contemporary'), ('Katy Perry: Part of Me', 'film.film.country', 'United States of America'), ('Demi Lovato', 'music.artist.genre', 'Teen pop'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Height'), ('181-beat', 'broadcast.content.genre', 'Urban contemporary'), ('Van Halen', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('#thatPower', 'common.topic.article', 'm.0rytzw4'), ('Jordin Sparks', 'music.artist.genre', 'Rhythm and blues'), ('Ricky Nelson', 'people.person.profession', 'Singer-songwriter'), ('2010 MTV Video Music Awards', 'award.award_ceremony.awards_presented', 'm.0n1ykxp'), ('181-beat', 'broadcast.content.genre', 'Rap music'), ('Contemporary R&B', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('m.0yrk4gn', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('Miley Cyrus', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Harry Chester'), ('Justin Timberlake', 'music.artist.genre', 'Rhythm and blues'), ('New Kids on the Block', 'music.artist.genre', 'Rhythm and blues'), ('Fabolous', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will Smith', 'people.person.profession', 'Record producer'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('181-beat', 'broadcast.content.artist', 'Trick Daddy'), ('1Club.FM: Power', 'broadcast.content.artist', 'Jennifer Lopez'), ('Rodney Jerkins', 'music.composer.compositions', 'Die in Your Arms'), ('William Orbit', 'people.person.profession', 'Record producer'), ('Jaden Smith', 'music.artist.contribution', 'm.0vp7qr5'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Montell Jordan', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('RBMG Records', 'music.record_label.artist', 'Justin Bieber'), ('Gavin DeGraw', 'music.artist.genre', 'Rock music'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0pcnqnb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Gavin DeGraw'), ('m.07lkzw7', 'common.webpage.topic', 'Justin Bieber'), ('Blackstreet', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Chrisette Michele'), ('Live My Life', 'music.single.versions', 'Live My Life (Jaywalker remix)'), ('Runaway Love (remix)', 'music.album.album_content_type', 'Remix album'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Fergie', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Janet Jackson', 'music.artist.label', 'The Island Def Jam Music Group'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njvtth'), ('Fergie', 'people.person.gender', 'Female'), ('Snoop Dogg', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lil Jon'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('Live My Life (Party Rock remix)', 'music.recording.song', 'Live My Life'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Eminem'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Kid Cudi'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0n58kgb'), ('1Club.FM: V101', 'common.topic.notable_types', 'Broadcast Content'), ('#Thatpower', 'music.recording.artist', 'Justin Bieber'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.producer', '1Club.FM'), ('m.0gxnnwp', 'people.sibling_relationship.sibling', 'Justin Bieber'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Lil Wayne', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Pattie Mallette', 'film.person_or_entity_appearing_in_film.films', 'm.0101fszs'), ('Wait For a Minute', 'music.recording.tracks', 'Wait For a Minute'), ('J. Holiday', 'freebase.valuenotation.has_value', 'Parents'), ('Yves Bole', 'base.fashionmodels.fashion_model.hair_color', 'Brown'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'MTV Films'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('Alicia Keys', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Electronic music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Sean Kingston'), ('Beauty And A Beat', 'award.award_nominated_work.award_nominations', 'm.0_vmmj6'), ('m.0101fv7m', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0d33hsc', 'celebrities.friendship.friend', 'Selena Gomez'), ('Never Let You Go', 'music.recording.song', 'Never Let You Go'), ('50 Cent', 'broadcast.artist.content', 'PowerHitz'), ('Next to You', 'music.recording.artist', 'Justin Bieber'), ('Sir Mix-a-Lot', 'music.artist.genre', 'Hip hop music'), ('Alicia Keys', 'broadcast.artist.content', 'radioIO Todays POP'), ('Rudolph Valentino', 'common.topic.image', 'Rudolph Valentino'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Avril Lavigne'), ('Sir Mix-a-Lot', 'broadcast.artist.content', '.977 The Hits Channel'), ('Big R Radio - The Hawk', 'broadcast.content.artist', '3 Doors Down'), ('m.0ywtfj6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Victoria Justice', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5t8gm'), ('m.0z8s_wn', 'award.award_honor.award_winner', 'Justin Bieber'), ('Bad Day', 'common.topic.notable_types', 'Musical Album'), ('Electronic dance music', 'common.topic.notable_types', 'Musical genre'), ('radioIO Todays POP', 'broadcast.content.genre', 'Classic hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cory Gunz'), ('Wait For a Minute', 'common.topic.notable_for', 'g.1yghcn7b9'), ('Justin Bieber', 'music.artist.genre', 'Teen pop'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mario Winans'), ('m.0ndc3_1', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('K-Ci & JoJo', 'music.artist.genre', 'Contemporary R&B'), ('m.0y7y53r', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrkr34', 'award.award_honor.award', 'Teen Choice Award for Choice Twitter Personality'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Kelly Clarkson'), ('Ginuwine', 'music.artist.genre', 'Hip hop music'), ('Next to You', 'music.recording.featured_artists', 'Justin Bieber'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Boyfriend (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Will i Am', 'music.artist.album', '#thatPOWER'), ('Never Let You Go', 'common.topic.article', 'm.0bbvzq6'), ('Ashley Tisdale', 'broadcast.artist.content', 'radioIO Todays POP'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Justin Bieber'), ('Artist', 'common.topic.notable_types', 'Profession'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Jeremy Bieber', 'freebase.valuenotation.has_value', 'Parents'), ('Under the Mistletoe', 'music.album.genre', 'Teen pop'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Johnny Crawford', 'people.person.nationality', 'United States of America'), ('m.0101ftn8', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Spouse', 'rdf-schema#domain', 'Marriage'), ('m.0w5l5h1', 'freebase.valuenotation.has_no_value', 'Degree'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Wait For a Minute', 'common.topic.notable_for', 'g.1ygjd8zbs'), ('m.0gctps1', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.012bq275', 'celebrities.friendship.friend', 'Yves Bole'), ('Montell Jordan', 'people.person.profession', 'Record producer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mars Plastic'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Confident', 'music.album.release_type', 'Single'), ('Somebody to Love', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.artist.concert_tours', 'Believe Tour'), ('Janet Jackson', 'people.person.languages', 'English Language'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Don Henley', 'people.person.profession', 'Actor'), ('Parents', 'rdf-schema#range', 'Person')]\n" + ] + } + ], + "source": [ + "raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']]\n", + "print(raw_dataset_graphs[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To show the benefits of this indexer in action, we will use the following model to encode this sample of graphs using LargeGraphIndexer, along with naively." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuda\n" + ] + } + ], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we compare the clock times of encoding using both methods." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [02:01<00:00, 1.22s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "121.99496078491211\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Indexing question-by-question\n", + "dataset_graphs_embedded = []\n", + "start = time.time()\n", + "for graph in tqdm.tqdm(raw_dataset_graphs):\n", + " nodes_map = dict()\n", + " edges_map = dict()\n", + " edge_idx_base = []\n", + "\n", + " for src, edge, dst in graph:\n", + " # Collect nodes\n", + " if src not in nodes_map:\n", + " nodes_map[src] = len(nodes_map)\n", + " if dst not in nodes_map:\n", + " nodes_map[dst] = len(nodes_map)\n", + " \n", + " # Collect edge types\n", + " if edge not in edges_map:\n", + " edges_map[edge] = len(edges_map)\n", + "\n", + " # Record edge\n", + " edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst]))\n", + " \n", + " # Encode nodes and edges\n", + " sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x]))\n", + " sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x]))\n", + "\n", + " x = model.encode(sorted_nodes, batch_size=256)\n", + " edge_attrs_map = model.encode(sorted_edges, batch_size=256)\n", + " \n", + " edge_attrs = []\n", + " edge_idx = []\n", + " for trip in edge_idx_base:\n", + " edge_attrs.append(edge_attrs_map[trip[1]])\n", + " edge_idx.append([trip[0], trip[2]])\n", + "\n", + " dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0)))\n", + " \n", + " \n", + "print(time.time()-start)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Indexing...\n", + "Time to create whole knowledge_graph: 114.14257955551147\n", + "Retrieving Subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 206.96it/s]\n", + "100%|██████████| 100/100 [00:01<00:00, 83.32it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "114.77728700637817\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Using LargeGraphIndexer to make one large knowledge graph\n", + "from torch_geometric.data.large_graph_indexer import EDGE_RELATION\n", + "\n", + "start = time.time()\n", + "all_triplets_together = chain.from_iterable(raw_dataset_graphs)\n", + "# Index as one large graph\n", + "print('Indexing...')\n", + "indexer = LargeGraphIndexer.from_triplets(all_triplets_together)\n", + "\n", + "# first the nodes\n", + "unique_nodes = indexer.get_unique_node_features()\n", + "node_encs = model.encode(unique_nodes, batch_size=256)\n", + "indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs)\n", + "\n", + "# then the edges\n", + "unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION)\n", + "edge_attr = model.encode(unique_edges, batch_size=256)\n", + "indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION)\n", + "\n", + "ckpt_time = time.time()\n", + "whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') \n", + "whole_graph_done = time.time()\n", + "print(f\"Time to create whole knowledge_graph: {whole_graph_done-start}\")\n", + "\n", + "# Compute this to make sure we're comparing like to like on final time printout\n", + "whole_graph_diff = whole_graph_done-ckpt_time\n", + "\n", + "# retrieve subgraphs\n", + "print('Retrieving Subgraphs...')\n", + "dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)]\n", + "print(time.time()-start-whole_graph_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The large graph indexer allows us to compute the entire knowledge graph from a series of samples, so that new retrieval methods can also be tested on the entire graph. We will see this attempted in practice later on." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's worth noting that, although the times are relatively similar right now, the speedup with largegraphindexer will be much higher as the size of the knowledge graph grows. This is due to the speedup being a factor of the number of unique nodes and edges in the graph." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]),\n", + " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]),\n", + " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]),\n", + " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]),\n", + " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]),\n", + " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]),\n", + " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]),\n", + " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]),\n", + " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]),\n", + " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]),\n", + " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]),\n", + " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]),\n", + " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]),\n", + " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]),\n", + " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]),\n", + " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]),\n", + " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", + " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]),\n", + " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]),\n", + " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]),\n", + " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]),\n", + " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]),\n", + " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]),\n", + " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]),\n", + " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]),\n", + " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]),\n", + " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]),\n", + " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]),\n", + " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]),\n", + " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]),\n", + " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]),\n", + " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]),\n", + " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]),\n", + " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]),\n", + " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]),\n", + " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", + " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]),\n", + " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]),\n", + " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]),\n", + " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]),\n", + " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]),\n", + " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]),\n", + " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]),\n", + " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]),\n", + " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]),\n", + " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]),\n", + " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]),\n", + " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]),\n", + " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]),\n", + " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]),\n", + " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]),\n", + " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]),\n", + " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]),\n", + " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]),\n", + " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]),\n", + " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]),\n", + " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]),\n", + " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]),\n", + " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]),\n", + " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]),\n", + " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]),\n", + " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]),\n", + " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]),\n", + " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]),\n", + " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]),\n", + " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]),\n", + " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]),\n", + " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]),\n", + " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]),\n", + " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]),\n", + " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", + " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", + " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]),\n", + " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]),\n", + " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]),\n", + " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]),\n", + " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]),\n", + " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]),\n", + " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]),\n", + " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]),\n", + " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]),\n", + " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]),\n", + " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]),\n", + " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]),\n", + " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", + " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", + " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]),\n", + " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset_graphs_embedded_largegraphindexer" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]),\n", + " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]),\n", + " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]),\n", + " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]),\n", + " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]),\n", + " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]),\n", + " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]),\n", + " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]),\n", + " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]),\n", + " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]),\n", + " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]),\n", + " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]),\n", + " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]),\n", + " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]),\n", + " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]),\n", + " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]),\n", + " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", + " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]),\n", + " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]),\n", + " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]),\n", + " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]),\n", + " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]),\n", + " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]),\n", + " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]),\n", + " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]),\n", + " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]),\n", + " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]),\n", + " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]),\n", + " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]),\n", + " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]),\n", + " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]),\n", + " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]),\n", + " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]),\n", + " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]),\n", + " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]),\n", + " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", + " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]),\n", + " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]),\n", + " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]),\n", + " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]),\n", + " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]),\n", + " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]),\n", + " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]),\n", + " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]),\n", + " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]),\n", + " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]),\n", + " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]),\n", + " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]),\n", + " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]),\n", + " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]),\n", + " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]),\n", + " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]),\n", + " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]),\n", + " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]),\n", + " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]),\n", + " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]),\n", + " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]),\n", + " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]),\n", + " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]),\n", + " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]),\n", + " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]),\n", + " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]),\n", + " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]),\n", + " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]),\n", + " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]),\n", + " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]),\n", + " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]),\n", + " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", + " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", + " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]),\n", + " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]),\n", + " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]),\n", + " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]),\n", + " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]),\n", + " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]),\n", + " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]),\n", + " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]),\n", + " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]),\n", + " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]),\n", + " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]),\n", + " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]),\n", + " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", + " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", + " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]),\n", + " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset_graphs_embedded" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We expect the two results to be functionally identical, with the differences being due to floating point jitter." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", + " def _sorted_tensors_are_close(tensor1, tensor2):\n", + " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", + " def _graphs_are_same(tensor1, tensor2):\n", + " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", + " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", + " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", + " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/100 [00:00 2\u001b[0m ds_updated \u001b[38;5;241m=\u001b[39m \u001b[43mUpdatedWebQSPDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mupdated_dataset\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(time\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;241m-\u001b[39mstart)\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:182\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, whole_graph_retrieval, limit, override)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moverride \u001b[38;5;241m=\u001b[39m override\n\u001b[1;32m 180\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\n\u001b[1;32m 181\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 182\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_raw_data()\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:328\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]):\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncoding graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 328\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 330\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:260\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset._build_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;66;03m# Nodes:\u001b[39;00m\n\u001b[1;32m 259\u001b[0m nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindexer\u001b[38;5;241m.\u001b[39mget_unique_node_features()\n\u001b[0;32m--> 260\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindexer\u001b[38;5;241m.\u001b[39madd_node_feature(new_feature_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mx\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_feature_vals\u001b[38;5;241m=\u001b[39mx)\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# Edges:\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/nn/nlp/sentence_transformer.py:75\u001b[0m, in \u001b[0;36mSentenceTransformer.encode\u001b[0;34m(self, text, batch_size, output_device)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m start \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(text), batch_size):\n\u001b[1;32m 65\u001b[0m token \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtokenizer(\n\u001b[1;32m 66\u001b[0m text[start:start \u001b[38;5;241m+\u001b[39m batch_size],\n\u001b[1;32m 67\u001b[0m padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 68\u001b[0m truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 69\u001b[0m return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 70\u001b[0m )\n\u001b[1;32m 72\u001b[0m emb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m 73\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 74\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m---> 75\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_device\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcpu\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 77\u001b[0m embs\u001b[38;5;241m.\u001b[39mappend(emb)\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat(embs, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(embs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m embs[\u001b[38;5;241m0\u001b[39m]\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "start = time.time()\n", + "ds_updated = UpdatedWebQSPDataset('updated_dataset', limit=demo_number_of_graphs)\n", + "updated_time = time.time() - start\n", + "print(time.time()-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.bar(['WebQSP', 'UpdatedQSP'], [web_qsp_time,, updated_time])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we will take a set of multi-hop questions, and combine them with an existing Wikidata knowledge graph to produce a new dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To be continued in 0.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/llm_plus_gnn/doc/_Introduction.ipynb b/examples/llm_plus_gnn/doc/_Introduction.ipynb new file mode 100644 index 000000000000..027eccf19028 --- /dev/null +++ b/examples/llm_plus_gnn/doc/_Introduction.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Working with GNN RAG in Pytorch Geometric" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This series of documents and demos aims to provide a starting point and some experimental results for multi-step Retrieval Augmented Generation (RAG) using Graph Neural Networks." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Motivation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As Large Language Models (LLMs) quickly grow to dominate industry, they are increasingly being deployed at scale in use cases that require very specific contextual expertise. LLMs often struggle with these cases out of the box, as they will hallucinate answers that are not included in their training data. At the same time, many business already have large graph databases full of important context that can provide important domain-specific context to reduce hallucination and improve answer fidelity for LLMs. Graph Neural Networks (GNNs) provide a means for efficiently encoding this contextual information into the model, which can help LLMs to better understand and generate answers. Hence, theres an open research question as to how to effectively use GNN encodings efficiently for this purpose, that the tooling provided here can help investigate." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Architecture" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To model the use-case of RAG from a large knowledge graph of millions of nodes, we present the following architecture:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import SVG" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "svg_file_path = './media/flowchart.svg'\n", + "SVG(svg_file_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Graph RAG as shown in the diagram above follows the following order of operations:\n", + "\n", + "0. To start, not pictured here, there must exist a large knowledge graph that exists as a source of truth. The nodes and edges of this knowledge graph \n", + "\n", + "During inference time, RAG implementations that follow this architecture are composed of the following steps:\n", + "\n", + "1. Tokenize and encode the query using the LLM Encoder\n", + "2. Retrieve a subgraph of the larger knowledge graph (KG) relevant to the query and encode it using a GNN\n", + "3. Jointly embed the GNN embedding with the LLM embedding\n", + "4. Utilize LLM Decoder to decode joint embedding and generate a response" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each notebook that follows will highlight a different component of this pipeline." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/llm_plus_gnn/doc/media/flowchart.svg b/examples/llm_plus_gnn/doc/media/flowchart.svg new file mode 100644 index 000000000000..8d24d4a2c309 --- /dev/null +++ b/examples/llm_plus_gnn/doc/media/flowchart.svg @@ -0,0 +1 @@ + \ No newline at end of file From d58568e5a487e07264c9bae9a0a2476becde934f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 14 Aug 2024 18:33:00 -0700 Subject: [PATCH 647/752] multihop --- ...0_0_Encoding_a_Large_Knowledge_Graph.ipynb | 397 +++ .../doc/0_1_Encoding_from_Scratch.ipynb | 836 +++++++ .../0_Encoding_a_Large_Knowledge_Graph.ipynb | 2143 ----------------- .../doc/media/multihop_example.svg | 1 + 4 files changed, 1234 insertions(+), 2143 deletions(-) create mode 100644 examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb create mode 100644 examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb delete mode 100644 examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb create mode 100644 examples/llm_plus_gnn/doc/media/multihop_example.svg diff --git a/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb b/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb new file mode 100644 index 000000000000..0b1993b71dec --- /dev/null +++ b/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb @@ -0,0 +1,397 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Encoding a Large Knowledge Graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we are going to walk through how to encode a large knowledge graph for the purposes of Graph RAG. We will provide two examples of how to do so, along with demonstration code." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 1: Building from Already Existing Datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. The same is true for GNN based RAG. Consider the following dataset WebQSP:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset\n", + "\n", + "# Computationally expensive, so running a small sample for now to show off the schema\n", + "# ds = WebQSPDataset('dataset')\n", + "num_questions = 100\n", + "ds = UpdatedWebQSPDataset('small_sample', limit=num_questions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "WebQSP is a dataset that is based off of a subset of the Freebase Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So each entry in the dataset consists of:\n", + "- A question to be answered\n", + "- The answer\n", + "- A knowledge graph subgraph of Freebase that has the context needed to answer the question." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds.raw_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds.raw_dataset[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Although this dataset can be trained on as-is, a couple problems emerge from doing so:\n", + "1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond to the algorithm that was used to generate the dataset subgraphs.\n", + "2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a result, it makes sense in this scenario to be able to encode all the entries into a large knowledge graph, so that duplicate nodes and edges can be avoided, and so that alternative retrieval algorithms can be tried. We can do this with the LargeGraphIndexer class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "import time\n", + "import torch\n", + "import tqdm\n", + "from itertools import chain\n", + "import networkx as nx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']]\n", + "print(raw_dataset_graphs[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To show the benefits of this indexer in action, we will use the following model to encode this sample of graphs using LargeGraphIndexer, along with naively." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we compare the clock times of encoding using both methods." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Indexing question-by-question\n", + "dataset_graphs_embedded = []\n", + "start = time.time()\n", + "for graph in tqdm.tqdm(raw_dataset_graphs):\n", + " nodes_map = dict()\n", + " edges_map = dict()\n", + " edge_idx_base = []\n", + "\n", + " for src, edge, dst in graph:\n", + " # Collect nodes\n", + " if src not in nodes_map:\n", + " nodes_map[src] = len(nodes_map)\n", + " if dst not in nodes_map:\n", + " nodes_map[dst] = len(nodes_map)\n", + " \n", + " # Collect edge types\n", + " if edge not in edges_map:\n", + " edges_map[edge] = len(edges_map)\n", + "\n", + " # Record edge\n", + " edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst]))\n", + " \n", + " # Encode nodes and edges\n", + " sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x]))\n", + " sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x]))\n", + "\n", + " x = model.encode(sorted_nodes, batch_size=256)\n", + " edge_attrs_map = model.encode(sorted_edges, batch_size=256)\n", + " \n", + " edge_attrs = []\n", + " edge_idx = []\n", + " for trip in edge_idx_base:\n", + " edge_attrs.append(edge_attrs_map[trip[1]])\n", + " edge_idx.append([trip[0], trip[2]])\n", + "\n", + " dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0)))\n", + " \n", + " \n", + "print(time.time()-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Using LargeGraphIndexer to make one large knowledge graph\n", + "from torch_geometric.data.large_graph_indexer import EDGE_RELATION\n", + "\n", + "start = time.time()\n", + "all_triplets_together = chain.from_iterable(raw_dataset_graphs)\n", + "# Index as one large graph\n", + "print('Indexing...')\n", + "indexer = LargeGraphIndexer.from_triplets(all_triplets_together)\n", + "\n", + "# first the nodes\n", + "unique_nodes = indexer.get_unique_node_features()\n", + "node_encs = model.encode(unique_nodes, batch_size=256)\n", + "indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs)\n", + "\n", + "# then the edges\n", + "unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION)\n", + "edge_attr = model.encode(unique_edges, batch_size=256)\n", + "indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION)\n", + "\n", + "ckpt_time = time.time()\n", + "whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') \n", + "whole_graph_done = time.time()\n", + "print(f\"Time to create whole knowledge_graph: {whole_graph_done-start}\")\n", + "\n", + "# Compute this to make sure we're comparing like to like on final time printout\n", + "whole_graph_diff = whole_graph_done-ckpt_time\n", + "\n", + "# retrieve subgraphs\n", + "print('Retrieving Subgraphs...')\n", + "dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)]\n", + "print(time.time()-start-whole_graph_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The large graph indexer allows us to compute the entire knowledge graph from a series of samples, so that new retrieval methods can also be tested on the entire graph. We will see this attempted in practice later on." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's worth noting that, although the times are relatively similar right now, the speedup with largegraphindexer will be much higher as the size of the knowledge graph grows. This is due to the speedup being a factor of the number of unique nodes and edges in the graph." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_graphs_embedded_largegraphindexer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_graphs_embedded" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We expect the two results to be functionally identical, with the differences being due to floating point jitter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", + " def _sorted_tensors_are_close(tensor1, tensor2):\n", + " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", + " def _graphs_are_same(tensor1, tensor2):\n", + " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", + " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", + " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", + " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_results_match = True\n", + "for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions):\n", + " all_results_match &= results_are_close_enough(old_graph, new_graph)\n", + "all_results_match" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When scaled up to the entire dataset, we see a 2x speedup with indexing this way.\n", + "\n", + "WebQSPDataset is a question-by-question implementation.\n", + "\n", + "UpdatedQSPDataset is a LargeGraphIndexer implementation.\n", + "\n", + "These were computed on an RTX 4090 with 24GB of memory. Your milage may vary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "demo_number_of_graphs = 4700 # out of 4700" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note: WebQSP doesn't let you limit the number of questions, so we compute the whole dataset and estimate the time by assuming each question takes the same amount of time\n", + "start = time.time()\n", + "ds_basic = WebQSPDataset('basic_dataset')\n", + "web_qsp_time = (time.time() - start)*demo_number_of_graphs/4700\n", + "print(time.time()-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start = time.time()\n", + "ds_updated = UpdatedWebQSPDataset('updated_dataset', limit=demo_number_of_graphs)\n", + "updated_time = time.time() - start\n", + "print(time.time()-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.bar(['WebQSP', 'UpdatedQSP'], [web_qsp_time,, updated_time])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we will take a set of multi-hop questions, and combine them with an existing Wikidata knowledge graph to produce a new dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To be continued in 0.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb b/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb new file mode 100644 index 000000000000..0188a025ba92 --- /dev/null +++ b/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb @@ -0,0 +1,836 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Encoding A Large Knowledge Graph Part 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we will continue where we left off by building a new multi-hop QA dataset based on Wikidata." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Motivation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One potential application of knowledge graph structural encodings is capturing the relationships between different entities that are multiple hops apart. This can be challenging for an LLM to recognize from prepended graph information. Here's a motivating example (credit to @Rishi Puri):" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import SVG" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "SVG(filename='./media/multihop_example.svg')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, the question can only be answered by reasoning about the relationships between the entities in the knowledge graph." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Building a Multi-Hop QA Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To start, we need to download the raw data of a knowledge graph. In this case, we use WikiData5M ([Wang et al](https://paperswithcode.com/paper/kepler-a-unified-model-for-knowledge)). Here we download the raw triplets and their entity codes. Information about this dataset can be found [here](https://deepgraphlearning.github.io/project/wikidata5m)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following download contains the ID to plaintext mapping for all the entities and relations in the knowledge graph:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!wget -O \"https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!tar -xvf \"wikidata5m_alias.tar.gz\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('wikidata5m_entity.txt') as f:\n", + " print(f.readline())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('wikidata5m_relation.txt') as f:\n", + " print(f.readline())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And then this download contains the raw triplets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!wget -O \"https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gzip -d \"wikidata5m_all_triplet.txt.gz\" -f" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('wikidata5m_all_triplet.txt') as f:\n", + " print(f.readline())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To start, we are going to preprocess the knowledge graph to substitute each of the entity/relation codes with their plaintext aliases. This makes it easier to use a pre-trained textual encoding model to create triplet embeddings, as such a model likely won't understand how to properly embed the entity codes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import tqdm\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Substitute entity codes with their aliases\n", + "# Picking the first alias for each entity (rather arbitrarily)\n", + "alias_map = {}\n", + "rel_alias_map = {}\n", + "for line in open('wikidata5m_entity.txt'):\n", + " parts = line.strip().split('\\t')\n", + " entity_id = parts[0]\n", + " aliases = parts[1:]\n", + " alias_map[entity_id] = aliases[0]\n", + "for line in open('wikidata5m_relation.txt'):\n", + " parts = line.strip().split('\\t')\n", + " relation_id = parts[0]\n", + " relation_name = parts[1]\n", + " rel_alias_map[relation_id] = relation_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "full_graph = []\n", + "missing_total = 0\n", + "total = 0\n", + "for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')):\n", + " src, rel, dst = line.strip().split('\\t')\n", + " if src not in alias_map:\n", + " missing_total += 1\n", + " if dst not in alias_map:\n", + " missing_total += 1\n", + " if rel not in rel_alias_map:\n", + " missing_total += 1\n", + " total += 3\n", + " full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)])\n", + "print(f\"Missing aliases: {missing_total}/{total}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "full_graph[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now `full_graph` represents the knowledge graph triplets in understandable plaintext." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we need a set of multi-hop questions that the Knowledge Graph will provide us with context for. We utilize a subset of [HotPotQA](https://hotpotqa.github.io/) ([Yang et. al.](https://arxiv.org/pdf/1809.09600)) called [2WikiMultiHopQA](https://github.com/Alab-NII/2wikimultihop) ([Ho et. al.](https://aclanthology.org/2020.coling-main.580.pdf)), which includes a subgraph of entities that serve as the ground truth justification for answering each multi-hop question:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!wget -O \"https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!unzip -o \"data_ids_april7.zip\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('train.json') as f:\n", + " train_data = json.load(f)\n", + "train_df = pd.DataFrame(train_data)\n", + "train_df['split_type'] = 'train'\n", + "\n", + "with open('dev.json') as f:\n", + " dev_data = json.load(f)\n", + "dev_df = pd.DataFrame(dev_data)\n", + "dev_df['split_type'] = 'dev'\n", + "\n", + "with open('test.json') as f:\n", + " test_data = json.load(f)\n", + "test_df = pd.DataFrame(test_data)\n", + "test_df['split_type'] = 'test'\n", + "\n", + "df = pd.concat([train_df, dev_df, test_df])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['split_type'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['type'].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we need to extract the subgraphs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['graph_size'] = df['evidences_id'].apply(lambda row: len(row))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['graph_size'].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(Optional) We take only questions where the evidence graph is greater than 0. (Note: this gets rid of the test set):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# df = df[df['graph_size'] > 0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df['split_type'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.columns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Checkpoint:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df.to_csv('wikimultihopqa_refined.csv', index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we need to check that all the entities mentioned in the question/answer set are also present in the Wikidata graph:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "relation_map = {}\n", + "with open('wikidata5m_relation.txt') as f:\n", + " for line in tqdm.tqdm(f):\n", + " parts = line.strip().split('\\t')\n", + " for i in range(1, len(parts)):\n", + " if parts[i] not in relation_map:\n", + " relation_map[parts[i]] = []\n", + " relation_map[parts[i]].append(parts[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M\n", + "for row in refined_df.itertuples():\n", + " for trip in row.evidences_id:\n", + " relation = trip[1]\n", + " if relation not in relation_map:\n", + " print(f'The following relation is not found: {relation}')\n", + " elif len(relation_map[relation]) > 1:\n", + " print(f'The following relation alias has a collision: {relation}: {relation_map[relation]}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "entity_set = set()\n", + "with open('wikidata5m_entity.txt') as f:\n", + " for line in tqdm.tqdm(f):\n", + " entity_set.add(line.strip().split('\\t')[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_entities = set()\n", + "missing_entity_idx = set()\n", + "for i, row in enumerate(refined_df.itertuples()):\n", + " for trip in row.evidences_id:\n", + " if len(trip) != 3:\n", + " print(trip)\n", + " entities = trip[0], trip[2]\n", + " for entity in entities:\n", + " if entity not in entity_set:\n", + " print(f'The following entity was not found in the KG: {entity}')\n", + " missing_entities.add(entity)\n", + " missing_entity_idx.add(i)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Right now, we drop the missing entity entries. Additional preprocessing can be done here to resolve the entity/relation collisions, but that is out of the scope for this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(missing_entity_idx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refined_df.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped.\n", + "refined_df.reset_index(inplace=True, drop=True)\n", + "refined_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cleaned_df = refined_df.drop(missing_entity_idx)\n", + "cleaned_df['split_type'].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we save the resulting graph and questions/answers dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "torch.save(full_graph, 'wikimultihopqa_full_graph.pt')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Question: How do we extract a contextual subgraph for a given query?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The chosen retrieval algorithm is a critical component in the pipeline for affecting RAG performance. In the next section (1), we will demonstrate a naive method of retrieval for a large knowledge graph, and how to apply it to this dataset along with WebQSP." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Preparing a Textualized Graph for LLM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For now however, we need to prepare the graph data to be used as a plaintext prefix to the LLM. In order to do this, we want to prompt the LLM to use the unique nodes, and unique edge triplets of a given subgraph. In order to do this, we prepare a unique indexed node df and edge df for the knowledge graph now. This process occurs trivially with the LargeGraphIndexer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import LargeGraphIndexer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "indexer = LargeGraphIndexer.from_triplets(full_graph)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Node DF\n", + "textual_nodes = pd.DataFrame.from_dict(\n", + " {\"node_attr\": indexer.get_node_features()})\n", + "textual_nodes[\"node_id\"] = textual_nodes.index\n", + "textual_nodes = textual_nodes[[\"node_id\", \"node_attr\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "textual_nodes.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice how LargeGraphIndexer ensures that there are no duplicate indices:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Edge DF\n", + "textual_edges = pd.DataFrame(indexer.get_edge_features(),\n", + " columns=[\"src\", \"edge_attr\", \"dst\"])\n", + "textual_edges[\"src\"] = [\n", + " indexer._nodes[h] for h in textual_edges[\"src\"]\n", + "]\n", + "textual_edges[\"dst\"] = [\n", + " indexer._nodes[h] for h in textual_edges[\"dst\"]\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "textual_edges.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can save the result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False)\n", + "textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now were done! This knowledge graph and dataset will get used later on in Section 1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: Refactor everything below this point into its own notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generating Subgraphs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from profiling_utils import create_remote_backend_from_triplets\n", + "from rag_feature_store import SentenceTransformerFeatureStore\n", + "from rag_graph_store import NeighborSamplingRAGGraphStore\n", + "from torch_geometric.loader import RAGQueryLoader\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", + "from torch_geometric.data import get_features_for_triplets_groups, Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={\"batch_size\": 256}).load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graphs = torch.load('subg_results.pt')\n", + "graph_df = pd.read_csv(\"wikimultihopqa_cleaned.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_df['is_train'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_df['is_train'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyg-local-dev", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb b/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb deleted file mode 100644 index 5bedae2d8415..000000000000 --- a/examples/llm_plus_gnn/doc/0_Encoding_a_Large_Knowledge_Graph.ipynb +++ /dev/null @@ -1,2143 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Encoding a Large Knowledge Graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this notebook, we are going to walk through how to encode a large knowledge graph for the purposes of Graph RAG. We will provide two examples of how to do so, along with demonstration code." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 1: Building from Already Existing Datasets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. The same is true for GNN based RAG. Consider the following dataset WebQSP:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset\n", - "\n", - "# Computationally expensive, so running a small sample for now to show off the schema\n", - "# ds = WebQSPDataset('dataset')\n", - "num_questions = 100\n", - "ds = UpdatedWebQSPDataset('small_sample', limit=num_questions)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "WebQSP is a dataset that is based off of a subset of the Freebase Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So each entry in the dataset consists of:\n", - "- A question to be answered\n", - "- The answer\n", - "- A knowledge graph subgraph of Freebase that has the context needed to answer the question." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Dataset({\n", - " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", - " num_rows: 100\n", - "})" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.raw_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'WebQTrn-0',\n", - " 'question': 'what is the name of justin bieber brother',\n", - " 'answer': ['Jaxon Bieber'],\n", - " 'q_entity': ['Justin Bieber'],\n", - " 'a_entity': ['Jaxon Bieber'],\n", - " 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", - " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", - " ['Rudolph Valentino',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", - " ['Record producer',\n", - " 'music.performance_role.regular_performances',\n", - " 'm.012m1vf1'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", - " ['2011 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0yrkr34'],\n", - " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Stratford', 'location.location.containedby', 'Canada'],\n", - " ['Singer',\n", - " 'base.lightweight.profession.specialization_of',\n", - " 'Musicians and Singers'],\n", - " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", - " ['As Long As You Love Me (Audien dubstep mix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kevin Risto', 'people.person.gender', 'Male'],\n", - " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", - " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", - " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", - " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", - " ['American Music Award for Favorite Pop/Rock Album',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc259'],\n", - " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", - " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", - " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Billboard Music Award for Top Streaming Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0njhx1b'],\n", - " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", - " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", - " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", - " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.episode',\n", - " 'Results Show: Week 7'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['One Less Lonely Girl',\n", - " 'music.album.primary_release',\n", - " 'One Less Lonely Girl'],\n", - " ['Twista', 'people.person.gender', 'Male'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", - " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", - " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", - " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", - " ['Madonna', 'music.artist.genre', 'Pop music'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", - " ['m.0gbm3cg',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", - " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", - " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", - " ['MTV Video Music Award Japan for Best New Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhrwc'],\n", - " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", - " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", - " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['m.0gctwjk',\n", - " 'tv.tv_guest_role.episodes_appeared_in',\n", - " 'Series 2, Episode 3'],\n", - " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", - " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", - " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", - " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", - " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Beauty and a Beat',\n", - " 'music.single.versions',\n", - " 'Beauty and a Beat (Wideboys Club Mix)'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Bryan Adams',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", - " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", - " ['Dany Brillant', 'people.person.gender', 'Male'],\n", - " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", - " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", - " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'music.composition.form', 'Song'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0yrkr34'],\n", - " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", - " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", - " ['Record producer',\n", - " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", - " 'Haley James Scott'],\n", - " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", - " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['Kanye West', 'people.person.profession', 'Singer'],\n", - " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", - " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", - " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", - " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", - " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", - " ['RedOne', 'music.artist.genre', 'Rock music'],\n", - " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", - " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", - " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'music.recording.featured_artists',\n", - " 'Big Sean'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", - " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", - " ['Athan Grace', 'people.person.profession', 'Actor'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", - " ['All Around the World (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", - " ['Brandy Norwood',\n", - " 'influence.influence_node.influenced_by',\n", - " 'Whitney Houston'],\n", - " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['MTV Video Music Award for Artist to Watch',\n", - " 'award.award_category.winners',\n", - " 'm.0n1ykxp'],\n", - " ['Caitlin Beadles',\n", - " 'celebrities.celebrity.sexual_relationships',\n", - " 'm.0d33gyj'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audiobot instrumental)'],\n", - " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", - " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'The Mighty Mighty Bosstones'],\n", - " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", - " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", - " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Documentary film'],\n", - " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", - " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Amerie', 'music.artist.genre', 'Pop music'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", - " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep edit)'],\n", - " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Recovery', 'music.recording.song', 'Recovery'],\n", - " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", - " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", - " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", - " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", - " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", - " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", - " ['#thatPower', 'music.recording.song', '#thatPower'],\n", - " ['Children', 'rdf-schema#range', 'Person'],\n", - " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", - " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['2013 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0wjgqck'],\n", - " ['The Island Def Jam Music Group',\n", - " 'organization.organization.parent',\n", - " 'm.04q65lb'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Rusted Root'],\n", - " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['m.0z87d3n',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", - " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", - " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", - " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", - " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", - " ['Tampa', 'location.location.containedby', 'United States of America'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.compositions',\n", - " 'Love Never Felt So Good'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", - " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", - " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", - " ['Estelle', 'people.person.profession', 'Record producer'],\n", - " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", - " ['Chris Jasper', 'people.person.gender', 'Male'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", - " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", - " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", - " ['Snoop Dogg',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['David Nicksay', 'people.person.gender', 'Male'],\n", - " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", - " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Juno Awards of 2014',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0102z0vx'],\n", - " ['As Long As You Love Me (Audiobot remix)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", - " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", - " ['m.0_cyzs_',\n", - " 'celebrities.legal_entanglement.offense',\n", - " 'Driving under the influence'],\n", - " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", - " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", - " ['Mann', 'people.person.gender', 'Male'],\n", - " ['JoJo', 'people.person.gender', 'Female'],\n", - " ['Right Here (featuring Drake)',\n", - " 'music.recording.canonical_version',\n", - " 'Right Here'],\n", - " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", - " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0yrjynf',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", - " ['Pras', 'people.person.profession', 'Record producer'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", - " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", - " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", - " ['Marvin Isley',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", - " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['School Gyrls', 'film.film.language', 'English Language'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", - " ['Katy Perry', 'people.person.profession', 'Actor'],\n", - " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", - " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.featured_artists',\n", - " 'Redfoo'],\n", - " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", - " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", - " ['As Long as You Love Me (album version)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Justin Bieber: Just Getting Started',\n", - " 'book.written_work.author',\n", - " 'Justin Bieber'],\n", - " ['BeirutNights.com Radio',\n", - " 'broadcast.content.artist',\n", - " 'Marc Maris vs. Ramone'],\n", - " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", - " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", - " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", - " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.releases',\n", - " 'Love Never Felt So Good'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", - " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", - " ['Dance music',\n", - " 'broadcast.genre.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", - " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", - " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", - " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", - " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", - " ['Scooter Braun', 'people.person.gender', 'Male'],\n", - " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", - " ['Sir Nolan', 'people.person.gender', 'Male'],\n", - " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", - " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", - " ['Adam Messinger',\n", - " 'music.composer.compositions',\n", - " \"Turn to You (Mother's Day Dedication)\"],\n", - " ['m.0yrktlv',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Male Hottie'],\n", - " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", - " ['Iggy Azalea',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", - " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['m.0gxnnzy',\n", - " 'celebrities.romantic_relationship.relationship_type',\n", - " 'Dated'],\n", - " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", - " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", - " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Britney Spears',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", - " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", - " ['Fergie', 'music.artist.genre', 'Rock music'],\n", - " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", - " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", - " ['Mistletoe', 'music.album.release_type', 'Single'],\n", - " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", - " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.tracks',\n", - " 'Live My Life (Party Rock remix)'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['m.0njw4z2',\n", - " 'award.award_honor.award',\n", - " 'MTV Europe Music Award for Best Male'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", - " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", - " ['m.0gbm3c3',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Miley Cyrus'],\n", - " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", - " ['United States of America',\n", - " 'base.biblioness.bibs_topic.is_really',\n", - " 'United States of America'],\n", - " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", - " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", - " ['JoJo', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Next to You', 'music.album.release_type', 'Single'],\n", - " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", - " ['Willa Ford', 'people.person.languages', 'English Language'],\n", - " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", - " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", - " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", - " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Hot Wired Radio',\n", - " 'broadcast.content.broadcast',\n", - " 'Hot Wired Radio - 128kbps Stream'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", - " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0gbm3d9',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", - " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", - " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", - " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", - " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", - " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", - " ['Justin Timberlake',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Khalil', 'people.person.gender', 'Male'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", - " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", - " ['Selena Gomez',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", - " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", - " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['C1', 'music.artist.genre', 'Hip hop music'],\n", - " ['My Worlds Acoustic',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Album content type'],\n", - " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", - " ['Live My Life', 'music.composition.language', 'English Language'],\n", - " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", - " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", - " ['Baby', 'music.album.releases', 'Baby'],\n", - " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", - " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", - " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", - " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", - " ['Big R Radio - The Hawk',\n", - " 'broadcast.content.artist',\n", - " 'The Black Eyed Peas'],\n", - " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", - " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", - " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", - " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.production_companies',\n", - " 'AEG Live'],\n", - " ['Redfoo', 'people.person.gender', 'Male'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", - " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", - " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", - " ['m.01053qzf',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", - " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", - " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", - " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", - " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", - " ['Ray J', 'people.person.profession', 'Musician'],\n", - " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", - " ['m.0101fv5f',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", - " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", - " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", - " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", - " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.appearance_type',\n", - " 'Guest host'],\n", - " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", - " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", - " ['Christina Milian', 'people.person.profession', 'Actor'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", - " ['Dance music', 'broadcast.genre.content', '181-party'],\n", - " ['Queen Elizabeth II Diamond Jubilee Medal',\n", - " 'award.award_category.winners',\n", - " 'm.0njwqrb'],\n", - " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", - " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", - " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['CMT Music Award: Collaborative Video of the Year',\n", - " 'award.award_category.winners',\n", - " 'm.0njvs9s'],\n", - " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", - " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", - " ['Chingy', 'people.person.profession', 'Actor'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", - " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", - " ['Justin Bieber',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", - " ['Chingy', 'people.person.gender', 'Male'],\n", - " ['Reed Smoot', 'people.person.gender', 'Male'],\n", - " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", - " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.releases',\n", - " 'As Long As You Love Me (remixes)'],\n", - " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjvlh'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", - " ['Singer', 'common.topic.article', 'm.09l6h'],\n", - " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", - " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'award.award_winning_work.awards_won',\n", - " 'm.0pc670l'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", - " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", - " ['Beyoncé Knowles',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['m.0njhyh_',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Streaming Song (Video)'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", - " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_production_design_by',\n", - " 'Devorah Herbert'],\n", - " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", - " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", - " ['Nick Jonas',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", - " ['Lupe Fiasco',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['Martin Kierszenbaum',\n", - " 'people.person.place_of_birth',\n", - " 'United States of America'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long as You Love Me'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", - " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", - " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", - " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", - " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", - " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['m.0y5tl39',\n", - " 'film.personal_film_appearance.film',\n", - " 'Les Coulisses des Golden Globes'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", - " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", - " ['justinbieber',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0z0tgz6'],\n", - " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", - " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", - " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", - " ['Everlast', 'music.artist.label', 'Island Records'],\n", - " ['5th Annual Shorty Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0ywvh8k'],\n", - " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Baby', 'common.topic.notable_types', 'Composition'],\n", - " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['m.0tkqqgg',\n", - " 'award.award_nomination.award',\n", - " 'Juno Award for Pop Album of the Year'],\n", - " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", - " ['Person', 'type.type.properties', 'Parents'],\n", - " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Nasri', 'people.person.profession', 'Singer'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.compositions',\n", - " 'As Long as You Love Me'],\n", - " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", - " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", - " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", - " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", - " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", - " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", - " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", - " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", - " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", - " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", - " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.film',\n", - " 'Zendaya: Behind the Scenes'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", - " ['That Should Be Me', 'music.composition.form', 'Song'],\n", - " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", - " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Justin Bieber'],\n", - " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.song',\n", - " 'Beauty And A Beat'],\n", - " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", - " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Usher', 'music.composer.compositions', 'First Dance'],\n", - " ['m.0n1ykxp',\n", - " 'award.award_honor.award',\n", - " 'MTV Video Music Award for Artist to Watch'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Rockumentary'],\n", - " ['Amerie', 'people.person.gender', 'Female'],\n", - " ['Real Change: Artists for Education',\n", - " 'film.film.personal_appearances',\n", - " 'm.0y5th3r'],\n", - " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", - " ['Beautiful and the Beat',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['PYD', 'common.topic.notable_types', 'Composition'],\n", - " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", - " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", - " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", - " ['iJustine', 'people.person.gender', 'Female'],\n", - " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", - " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", - " ['m.0ng_vkd',\n", - " 'film.personal_film_appearance.film',\n", - " 'A Michael Bublé Christmas'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", - " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", - " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", - " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Vanessa Hudgens',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", - " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", - " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Madonna', 'people.person.profession', 'Record producer'],\n", - " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", - " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", - " ['As Long as You Love Me (acoustic version)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", - " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", - " ['Rob Thomas', 'people.person.gender', 'Male'],\n", - " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", - " ['m.0hvlt03',\n", - " 'film.film_film_distributor_relationship.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", - " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['justinbieber',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0_srv2b'],\n", - " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", - " ['Donna Summer',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['m.0101fszs',\n", - " 'film.personal_film_appearance.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Alanis Morissette',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Official website'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Jenna Andrews'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", - " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Ray J', 'people.person.nationality', 'United States of America'],\n", - " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Nelly Furtado',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['m.0gbm3fj',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Bryan-Michael Cox',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", - " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", - " ['As Long As You Love Me (Ferry Corsten club dub)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", - " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", - " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", - " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", - " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", - " ['Guest host',\n", - " 'tv.non_character_role.tv_guest_personal_appearances',\n", - " 'm.0v_98y7'],\n", - " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", - " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", - " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", - " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", - " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", - " ['L.A. Reid', 'people.person.gender', 'Male'],\n", - " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['London', 'location.location.containedby', 'Ontario'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_set_decoration_by',\n", - " 'Lia Roldan'],\n", - " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", - " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", - " ['Children', 'type.property.schema', 'Person'],\n", - " ['Change Me', 'music.album.releases', 'Change Me'],\n", - " ['RedOne', 'music.artist.label', 'Island Records'],\n", - " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", - " ['All Around the World',\n", - " 'music.recording.canonical_version',\n", - " 'All Around the World'],\n", - " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0wjhc6c'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", - " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", - " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Lupe Fiasco'],\n", - " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", - " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", - " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", - " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", - " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Beauty and a Beat (Wideboys Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", - " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", - " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", - " ['m.0v90skf',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Male Artist'],\n", - " ['Ludacris', 'people.person.profession', 'Actor'],\n", - " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", - " ['Cameo appearance',\n", - " 'tv.special_tv_performance_type.episode_performances',\n", - " 'm.0v1lwt2'],\n", - " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", - " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", - " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'One Chance'],\n", - " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", - " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", - " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", - " ['Roller Coaster',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0_x4zg3'],\n", - " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", - " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", - " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", - " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", - " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", - " ['m.0sgk_cw',\n", - " 'award.award_honor.award',\n", - " \"Kids' Choice Award for Favorite Song\"],\n", - " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", - " ['MTV Video Music Brazil Award for Best International Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhhqv'],\n", - " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", - " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", - " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", - " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", - " ['Britney Spears', 'people.person.profession', 'Singer'],\n", - " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", - " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Rock music',\n", - " 'base.webvideo.internet_video_genre.series',\n", - " 'Biscuithands, The Animated Musical'],\n", - " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", - " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Barry Weiss'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['MTV Europe Music Award for Best Male',\n", - " 'award.award_category.winners',\n", - " 'm.0z1scxk'],\n", - " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", - " ['Will Smith', 'people.person.profession', 'Actor'],\n", - " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", - " ['Will i Am',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Fabian', 'people.person.gender', 'Male'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", - " ['Somebody to Love (remix)',\n", - " 'music.album.primary_release',\n", - " 'Somebody to Love (remix)'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", - " ['Spouse', 'type.property.expected_type', 'Person'],\n", - " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", - " ['Baby', 'music.recording.artist', 'Ludacris'],\n", - " ['Rudolph Valentino',\n", - " 'people.person.nationality',\n", - " 'United States of America'],\n", - " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", - " ['Judy Garland',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Kelly Clarkson',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'base.schemastaging.context_name.pronunciation',\n", - " 'm.011h9_22'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", - " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", - " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0vp8rhw',\n", - " 'music.recording_contribution.album',\n", - " 'Beauty and a Beat (Remixes)'],\n", - " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", - " ['Katy Perry: Part of Me',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['m.0jsmvv5',\n", - " 'film.film_regional_release_date.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", - " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", - " ['All Around The World (featuring Ludacris)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", - " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", - " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", - " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", - " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", - " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep mix)'],\n", - " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Fergie', 'people.person.profession', 'Actor'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", - " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", - " ['Zac Efron', 'people.person.gender', 'Male'],\n", - " ['P!nk', 'music.artist.genre', 'Rock music'],\n", - " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", - " ['Gender', 'type.property.schema', 'Person'],\n", - " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", - " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", - " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", - " ['Under the Mistletoe',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Initial release date'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Slick Rick'],\n", - " ['Amerie', 'music.artist.genre', 'Rock music'],\n", - " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0pbzq13',\n", - " 'film.performance.special_performance_type',\n", - " 'Cameo appearance'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", - " ['Height', 'type.property.unit', 'Meter'],\n", - " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", - " ['Ray J',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Boyfriend (acoustic version)',\n", - " 'music.recording.canonical_version',\n", - " 'Boyfriend'],\n", - " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Believe Tour',\n", - " 'music.concert_tour.album_or_release_supporting',\n", - " 'Believe'],\n", - " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", - " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", - " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", - " ['Frank Ocean',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['As Long As You Love Me (Audiobot instrumental)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Usher', 'people.person.nationality', 'United States of America'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", - " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", - " ['m.0gbm3b7',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Sheryl Crow',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['All That Matters',\n", - " 'music.composition.composer',\n", - " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Nasri', 'music.artist.genre', 'Reggae'],\n", - " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", - " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", - " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", - " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", - " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", - " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", - " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long As You Love Me'],\n", - " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Bryan-Michael Cox',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", - " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", - " ['Whitney Houston', 'people.person.profession', 'Model'],\n", - " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['As Long as You Love Me',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Disney Parks Christmas Day Parade',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['Ray J', 'people.person.profession', 'Artist'],\n", - " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", - " ['American Music Award for Favorite Pop/Rock Male Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc0sf'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", - " ['The Notorious B.I.G.',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", - " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", - " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", - " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", - " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Haley James Scott',\n", - " 'fictional_universe.fictional_character.occupation',\n", - " 'Record producer'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", - " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", - " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['My Worlds: The Collection',\n", - " 'music.album.album_content_type',\n", - " 'Compilation album'],\n", - " ['Lolly', 'music.album.releases', 'Lolly'],\n", - " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", - " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", - " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", - " ['Ja Rule', 'people.person.profession', 'Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", - " ['Die in Your Arms',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0z85qxq'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", - " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", - " ['Kuk Harrell',\n", - " 'film.person_or_entity_appearing_in_film.films',\n", - " 'm.0101ft5f'],\n", - " ['Somebody to Love (J Stax remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'film.film.executive_produced_by',\n", - " 'Allison Kaye Scarinzi'],\n", - " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", - " ['Nasri', 'music.artist.genre', 'Pop music'],\n", - " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", - " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", - " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", - " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", - " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.05sp405',\n", - " 'organization.organization_relationship.child',\n", - " 'Island Records'],\n", - " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", - " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", - " ['John Mamann',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Teen Choice Award for Choice Summer Music Star: Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjynf'],\n", - " ['Juicy J', 'people.person.profession', 'Actor'],\n", - " ['m.0yqflrk',\n", - " 'measurement_unit.dated_money_value.source',\n", - " 'celebritynetworth.com'],\n", - " ['Miley Cyrus',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", - " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['m.04q65lb',\n", - " 'organization.organization_relationship.child',\n", - " 'The Island Def Jam Music Group'],\n", - " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", - " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", - " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['My World', 'music.album.artist', 'Justin Bieber'],\n", - " ['Don Henley', 'people.person.gender', 'Male'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Musician', 'people.profession.specializations', 'Singer'],\n", - " ['Die in Your Arms',\n", - " 'music.recording.canonical_version',\n", - " 'Die in Your Arms'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['m.0njvs9s',\n", - " 'award.award_honor.award',\n", - " 'CMT Music Award: Collaborative Video of the Year'],\n", - " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber',\n", - " 'music.artist.album',\n", - " 'Turn to You (Mother’s Day Dedication)'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", - " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['City/Town/Village',\n", - " 'freebase.type_profile.strict_included_types',\n", - " 'Topic'],\n", - " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", - " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Terence Dudley', 'people.person.gender', 'Male'],\n", - " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['BeirutNights.com Radio',\n", - " 'broadcast.content.artist',\n", - " 'Mr. Sosa & The Yayo'],\n", - " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", - " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", - " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Gender', 'type.property.expected_type', 'Gender'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0101fvbf',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['m.0yrhrwc',\n", - " 'award.award_honor.ceremony',\n", - " '2011 MTV Video Music Aid Japan'],\n", - " ['MTV Europe Music Award for Best North American Act',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhmll'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['m.0101ft1d',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", - " ['Ciara', 'people.person.gender', 'Female'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", - " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", - " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0jzrrqs',\n", - " 'location.mailing_address.country',\n", - " 'United States of America'],\n", - " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", - " ['Big R Radio - Top 40 Hits',\n", - " 'broadcast.content.artist',\n", - " 'Natasha Bedingfield'],\n", - " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ...],\n", - " 'choices': []}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.raw_dataset[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Although this dataset can be trained on as-is, a couple problems emerge from doing so:\n", - "1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond to the algorithm that was used to generate the dataset subgraphs.\n", - "2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As a result, it makes sense in this scenario to be able to encode all the entries into a large knowledge graph, so that duplicate nodes and edges can be avoided, and so that alternative retrieval algorithms can be tried. We can do this with the LargeGraphIndexer class:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "import time\n", - "import torch\n", - "import tqdm\n", - "from itertools import chain\n", - "import networkx as nx" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkr34'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'), ('As Long As You Love Me (Ferry Corsten radio)', 'common.topic.notable_types', 'Musical Recording'), ('Toby Gad', 'music.artist.genre', 'Rhythm and blues'), ('Stratford', 'location.location.containedby', 'Canada'), ('Singer', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('Enrique Iglesias', 'people.person.profession', 'Singer'), ('Beauty and a Beat (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'), ('50 Cent', 'people.person.profession', 'Film Producer'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Kevin Risto', 'people.person.gender', 'Male'), ('Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'), ('Shaggy', 'broadcast.artist.content', 'HitzRadio.com'), ('Mary J. Blige', 'people.person.profession', 'Record producer'), ('Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'), ('Paul Anka', 'common.topic.notable_types', 'Musical Artist'), ('m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'), ('Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'), ('m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'), ('1Club.FM: V101', 'broadcast.content.artist', 'The Roots'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('American Music Award for Favorite Pop/Rock Album', 'award.award_category.winners', 'm.0ndc259'), ('A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'), ('Ontario', 'location.administrative_division.country', 'Canada'), ('1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'), ('Music Producer', 'common.topic.subject_of', 'POPPMusic.net'), ('Billboard Music Award for Top Streaming Artist', 'award.award_category.winners', 'm.0njhx1b'), ('Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"), ('Heartbreaker', 'music.composition.recordings', 'Heartbreaker'), ('Brandy Norwood', 'people.person.profession', 'Singer'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'), ('Justin Bieber', 'music.artist.album', 'All Bad'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('m.0v_729v', 'tv.tv_guest_personal_appearance.episode', 'Results Show: Week 7'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'), ('One Less Lonely Girl', 'music.album.primary_release', 'One Less Lonely Girl'), ('Twista', 'people.person.gender', 'Male'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'), ('Ciara', 'broadcast.artist.content', 'FLOW 103'), ('Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Somebody To Love', 'music.recording.artist', 'Justin Bieber'), ('Toby Gad', 'music.artist.genre', 'Rock music'), ('Madonna', 'music.artist.genre', 'Pop music'), ('Selena Gomez', 'music.artist.genre', 'Europop'), ('m.0gbm3cg', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Baby', 'music.recording.canonical_version', 'Baby'), ('Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Dr. Dre', 'music.artist.genre', 'Rap music'), ('MTV Video Music Award Japan for Best New Artist', 'award.award_category.winners', 'm.0yrhrwc'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Hip hop music', 'broadcast.genre.content', 'FLOW 103'), ('Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('m.0gctwjk', 'tv.tv_guest_role.episodes_appeared_in', 'Series 2, Episode 3'), ('Enrique Iglesias', 'music.artist.genre', 'Dance-pop'), ('Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'), ('FLOW 103', 'broadcast.content.genre', 'Hip hop music'), ('Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Selena Gomez', 'people.person.profession', 'Dancer'), ('Little Bird', 'music.recording.tracks', 'm.0v2hrym'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'), ('Never Say Never', 'common.topic.notable_types', 'Musical Recording'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Club Mix)'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Madonna', 'people.person.profession', 'Singer-songwriter'), ('Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'), ('Terence Dudley', 'music.artist.genre', 'Reggae'), ('Kylie Minogue', 'people.person.profession', 'Actor'), ('Adrienne Bailon', 'music.artist.genre', 'Pop music'), ('Katy Perry', 'music.artist.genre', 'Electronic music'), ('Dany Brillant', 'people.person.gender', 'Male'), ('Martin Kierszenbaum', 'people.person.gender', 'Male'), ('Anastacia', 'people.person.nationality', 'United States of America'), ('Amerie', 'music.artist.label', 'The Island Def Jam Music Group'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Children'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Somebody to Love', 'music.composition.form', 'Song'), ('Teen Choice Award for Choice Twitter Personality', 'award.award_category.winners', 'm.0yrkr34'), ('Chef Tone', 'people.person.place_of_birth', 'Chicago'), ('Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Haley James Scott'), ('Colbie Caillat', 'music.artist.genre', 'Pop music'), ('C1', 'music.artist.genre', 'Contemporary R&B'), ('Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'), ('Kanye West', 'people.person.profession', 'Singer'), ('Pop music', 'common.topic.subject_of', 'Stephen Melton'), ('radioIO Todays POP', 'broadcast.content.producer', 'Radioio'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'), ('Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'), ('Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'), ('Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'), ('Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'), ('Adam Messinger', 'music.composer.compositions', 'Mistletoe'), ('Live My Life', 'music.album.compositions', 'Live My Life'), ('RedOne', 'music.artist.genre', 'Rock music'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'), ('Terius Nash', 'music.artist.genre', 'Rhythm and blues'), ('Little Bird', 'common.topic.notable_types', 'Musical Recording'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.featured_artists', 'Big Sean'), ('Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Terius Nash', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'), ('Athan Grace', 'people.person.profession', 'Actor'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'), ('All Around the World (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Bad Day', 'music.composition.composer', 'Marvin Isley'), ('Brandy Norwood', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('MTV Video Music Award for Artist to Watch', 'award.award_category.winners', 'm.0n1ykxp'), ('Caitlin Beadles', 'celebrities.celebrity.sexual_relationships', 'm.0d33gyj'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot instrumental)'), ('Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'), ('School Boy Records', 'music.record_label.artist', 'Scooter Braun'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Mighty Mighty Bosstones'), ('m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'), ('Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'), ('Lolly', 'music.composition.composer', 'Juicy J'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Documentary film'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Amerie', 'music.artist.genre', 'Pop music'), ('1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'), ('Rodney Jerkins', 'music.artist.genre', 'Synthpop'), ('WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien dubstep edit)'), ('Will Smith', 'broadcast.artist.content', 'Sunshine Radio'), ('Recovery', 'music.recording.song', 'Recovery'), ('Justin Timberlake', 'music.artist.genre', 'Electronic music'), ('Mannie Fresh', 'people.person.nationality', 'United States of America'), ('m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Benny Blanco', 'common.topic.notable_types', 'Record Producer'), ('Leif Garrett', 'music.artist.genre', 'Rock music'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'), ('First Dance', 'music.recording.artist', 'Justin Bieber'), ('#thatPower', 'music.recording.song', '#thatPower'), ('Children', 'rdf-schema#range', 'Person'), ('Beautiful', 'common.topic.notable_for', 'g.1256glpl9'), ('Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjgqck'), ('The Island Def Jam Music Group', 'organization.organization.parent', 'm.04q65lb'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Rusted Root'), ('radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'), ('m.0z87d3n', 'award.award_honor.award', 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'), ('Shaffer Smith', 'music.artist.genre', 'Dance music'), ('Live My Life', 'music.composition.composer', 'John Mamann'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'), ('m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Trick Daddy', 'broadcast.artist.content', 'PowerHitz'), ('1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'), ('Tampa', 'location.location.containedby', 'United States of America'), ('Love Never Felt So Good', 'music.album.compositions', 'Love Never Felt So Good'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.artist', 'Justin Bieber'), ('Nelly', 'music.artist.genre', 'Rhythm and blues'), ('Marvin Isley', 'music.composer.compositions', 'Bad Day'), ('Somebody to Love', 'common.topic.notable_types', 'Composition'), ('Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'), ('Snoop Dogg', 'people.person.gender', 'Male'), ('DMX', 'broadcast.artist.content', '.977 The Hits Channel'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'), ('Estelle', 'people.person.profession', 'Record producer'), ('m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('PowerHitz', 'broadcast.content.genre', 'Hip hop music'), ('Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('50 Cent', 'people.person.nationality', 'United States of America'), ('Chris Jasper', 'people.person.gender', 'Male'), ('Sir Nolan', 'music.artist.genre', 'Pop music'), ('Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('David Nicksay', 'people.person.gender', 'Male'), ('Justin Bieber', 'people.person.profession', 'Record producer'), ('Everlast', 'people.person.profession', 'Singer-songwriter'), ('Juno Awards of 2014', 'award.award_ceremony.awards_presented', 'm.0102z0vx'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.song', 'As Long as You Love Me'), ('#thatPower', 'music.composition.composer', 'Will i Am'), ('m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'), ('m.0_cyzs_', 'celebrities.legal_entanglement.offense', 'Driving under the influence'), ('LeAnn Rimes', 'people.person.profession', 'Actor'), ('KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'), ('1Club.FM: Power', 'broadcast.content.artist', 'Usher'), ('Mann', 'people.person.gender', 'Male'), ('JoJo', 'people.person.gender', 'Female'), ('Right Here (featuring Drake)', 'music.recording.canonical_version', 'Right Here'), ('Mason Levy', 'music.composer.compositions', 'Boyfriend'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('m.0yrjynf', 'award.award_honor.award', 'Teen Choice Award for Choice Summer Music Star: Male'), ('Pras', 'people.person.profession', 'Record producer'), ('1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'), ('Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Raekwon', 'broadcast.artist.content', 'Smoothbeats'), ('Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'), ('Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'), ('My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('School Gyrls', 'film.film.language', 'English Language'), ('Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'), ('Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'), ('Katy Perry', 'people.person.profession', 'Actor'), ('As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'), ('Ronald Isley', 'people.person.profession', 'Actor'), ('Live My Life (Party Rock remix)', 'music.recording.featured_artists', 'Redfoo'), ('HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'), ('Jaxon Bieber', 'people.person.nationality', 'Canada'), ('As Long as You Love Me (album version)', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber: Just Getting Started', 'book.written_work.author', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Marc Maris vs. Ramone'), ('Gwen Stefani', 'people.person.profession', 'Musician'), ('m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'), ('Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'), ('Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'), ('Love Never Felt So Good', 'music.album.releases', 'Love Never Felt So Good'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Sir Mix-a-Lot', 'people.person.profession', 'Actor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'), ('Dance music', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('1Club.FM: V101', 'broadcast.content.location', 'Chicago'), ('Terius Nash', 'people.person.profession', 'Record producer'), ('Terence Dudley', 'people.person.profession', 'Record producer'), ('Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'), ('Baby', 'common.topic.notable_types', 'Award-Winning Work'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Scooter Braun', 'people.person.gender', 'Male'), ('Mistletoe', 'music.album.artist', 'Justin Bieber'), ('Sir Nolan', 'people.person.gender', 'Male'), ('My Worlds: The Collection', 'music.album.genre', 'Teen pop'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'), ('Synthpop', 'music.genre.parent_genre', 'K-pop'), ('Adam Messinger', 'music.composer.compositions', \"Turn to You (Mother's Day Dedication)\"), ('m.0yrktlv', 'award.award_honor.award', 'Teen Choice Award for Choice Male Hottie'), ('Kanye West', 'people.person.nationality', 'United States of America'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'), ('Juicy J', 'freebase.valuenotation.has_value', 'Parents'), ('JellyRadio.com', 'broadcast.content.artist', 'DMX'), ('HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'), ('m.0gxnnzy', 'celebrities.romantic_relationship.relationship_type', 'Dated'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'), ('radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'), ('m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Britney Spears', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Frank Ocean', 'music.artist.genre', 'Rhythm and blues'), ('Ludacris', 'music.artist.contribution', 'm.0vp800w'), ('Singer', 'common.topic.subject_of', 'Justin Bieber'), ('Fergie', 'music.artist.genre', 'Rock music'), ('Gas Pedal', 'common.topic.notable_types', 'Musical Recording'), ('Toby Gad', 'people.person.profession', 'Record producer'), ('All Around The World', 'music.composition.composer', 'Justin Bieber'), ('Mistletoe', 'music.album.release_type', 'Single'), ('Kid Cudi', 'people.person.profession', 'Film Producer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'), ('Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.artist', 'Justin Bieber'), ('m.0njw4z2', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), (\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'), ('Snoop Dogg', 'people.person.profession', 'Record producer'), ('Savan Kotecha', 'music.artist.genre', 'Dance-pop'), ('m.0gbm3c3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Rodney Jerkins', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Miley Cyrus'), ('Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), (\"Destiny's Child\", 'music.artist.genre', 'Pop music'), ('United States of America', 'base.biblioness.bibs_topic.is_really', 'United States of America'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.09xx941', 'common.webpage.topic', 'Teen idol'), ('Christina Milian', 'people.person.profession', 'Record producer'), ('JoJo', 'people.person.nationality', 'United States of America'), ('Kylie Minogue', 'music.artist.genre', 'Electronic dance music'), ('Next to You', 'music.album.release_type', 'Single'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('Willa Ford', 'people.person.languages', 'English Language'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All That Matters', 'music.composition.composer', 'Andre Harris'), ('Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'), ('Paul Anka', 'music.artist.genre', 'Pop music'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'), ('Caitlin Beadles', 'people.person.nationality', 'Canada'), ('m.0z8s_wn', 'award.award_honor.honored_for', 'My World'), ('Favorite Girl', 'common.topic.notable_types', 'Musical Album'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'), ('Avery', 'common.topic.notable_types', 'Musical Artist'), ('m.0gbm3d9', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Beyoncé Knowles', 'people.person.profession', 'Actor'), ('m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Tupac Shakur', 'people.person.profession', 'Actor'), ('Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'), ('Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'), ('Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Frank Ocean', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'music.composer.compositions', 'Baby'), ('Chance the Rapper', 'music.artist.genre', 'Hip hop music'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Khalil', 'people.person.gender', 'Male'), ('#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Selena Gomez', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'), ('Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'), ('m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'), ('Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'), ('Anastacia', 'music.artist.genre', 'Contemporary R&B'), ('C1', 'music.artist.genre', 'Hip hop music'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Album content type'), ('m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'), ('Live My Life', 'music.composition.language', 'English Language'), ('Vocals', 'music.instrument.instrumentalists', 'Aaliyah'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'), ('Baby', 'music.album.releases', 'Baby'), ('A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'), ('Right Here', 'music.recording.canonical_version', 'Right Here'), ('Justin Bieber', 'people.person.profession', 'Musician'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bigger', 'music.composition.composer', 'Waynne Nugent'), ('Home to Mama', 'music.composition.composer', 'Cody Simpson'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Thought Of You', 'music.composition.composer', 'Justin Bieber'), ('The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'), ('Singer', 'people.profession.specializations', 'Prima donna'), ('Alanis Morissette', 'people.person.profession', 'Record producer'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'), ('Record producer', 'common.topic.notable_for', 'g.1258k9617'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'), ('Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'), ('Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'AEG Live'), ('Redfoo', 'people.person.gender', 'Male'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'), ('WildFMRadio.com', 'broadcast.content.artist', '50 Cent'), ('Ronald Isley', 'music.artist.genre', 'Quiet Storm'), ('Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Right Here', 'music.album.featured_artists', 'Drake'), ('m.01053qzf', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Guglielmo Scilla', 'common.topic.notable_types', 'Person'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'), ('Jordan Pruitt', 'music.artist.genre', 'Pop music'), ('Mason Levy', 'music.artist.genre', 'Rhythm and blues'), ('Thought of You', 'common.topic.notable_types', 'Canonical Version'), ('Whitney Houston', 'people.person.profession', 'Record producer'), ('m.07lkzw7', 'common.webpage.category', 'Official Website'), ('Ray J', 'people.person.profession', 'Musician'), ('m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Enrique Iglesias', 'people.person.gender', 'Male'), ('m.0101fv5f', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('HitzRadio.com', 'broadcast.content.artist', 'Nelly'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('Selena Gomez', 'music.artist.genre', 'Teen pop'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'), ('Love Never Felt So Good', 'music.album.genre', 'Disco'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'), ('m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_729v', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'), ('Ja Rule', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('Christina Milian', 'people.person.profession', 'Actor'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Dance music', 'broadcast.genre.content', '181-party'), ('Queen Elizabeth II Diamond Jubilee Medal', 'award.award_category.winners', 'm.0njwqrb'), ('Sean Kingston', 'people.person.profession', 'Singer'), ('DMX', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'), ('CMT Music Award: Collaborative Video of the Year', 'award.award_category.winners', 'm.0njvs9s'), ('m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'), ('One Time', 'common.topic.notable_types', 'Musical Album'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'), ('Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'), ('Katy Perry', 'music.artist.genre', 'Disco'), ('Chingy', 'people.person.profession', 'Actor'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'), ('Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'), ('Rihanna', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Contemporary R&B', 'common.topic.notable_types', 'Musical genre'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'City High'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'), ('Chingy', 'people.person.gender', 'Male'), ('Reed Smoot', 'people.person.gender', 'Male'), (\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'), ('Teyana', 'freebase.valuenotation.has_value', 'Parents'), ('Next to You', 'music.recording.song', 'Next to You'), ('All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('As Long as You Love Me', 'music.album.releases', 'As Long As You Love Me (remixes)'), ('Teen Choice Award for Choice Music: Breakout Artist - Male', 'award.award_category.winners', 'm.0yrjvlh'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'), ('Singer', 'common.topic.article', 'm.09l6h'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'), ('Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber: Never Say Never', 'award.award_winning_work.awards_won', 'm.0pc670l'), ('1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'), ('Beauty And A Beat', 'music.composition.form', 'Song'), ('Britney Spears', 'music.artist.genre', 'Electronic dance music'), ('HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('m.0njhyh_', 'award.award_honor.award', 'Billboard Music Award for Top Streaming Song (Video)'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'), ('Ludacris', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Never Say Never', 'film.film.film_production_design_by', 'Devorah Herbert'), ('Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Drake', 'music.artist.genre', 'Rhythm and blues'), ('Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'), ('Lupe Fiasco', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Martin Kierszenbaum', 'people.person.place_of_birth', 'United States of America'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'), ('m.0d_hbgr', 'common.webpage.category', 'Lyrics'), ('Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'), ('Beautiful', 'music.composition.lyricist', 'Toby Gad'), ('Redfoo', 'music.artist.genre', 'Electronic dance music'), ('1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'), ('K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'), ('K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'), ('Stephen Melton', 'music.group_member.instruments_played', 'Vocals'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Contemporary R&B', 'broadcast.genre.content', '181-thebox'), ('Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'), ('m.0y5tl39', 'film.personal_film_appearance.film', 'Les Coulisses des Golden Globes'), ('Teen idol', 'common.topic.webpage', 'm.09y89l2'), ('m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kevin Risto', 'people.person.profession', 'Musician'), ('Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z0tgz6'), ('Justin Bieber', 'music.artist.label', 'Island Records'), ('Ernie Isley', 'people.person.nationality', 'United States of America'), ('Kylie Minogue', 'people.person.profession', 'Film Producer'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'), ('Everlast', 'music.artist.label', 'Island Records'), ('5th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0ywvh8k'), ('Chance the Rapper', 'music.featured_artist.albums', 'Confident'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'), ('Baby', 'common.topic.notable_types', 'Composition'), ('Fabian', 'base.icons.icon.icon_genre', 'Teen idol'), ('Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0tkqqgg', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Person', 'type.type.properties', 'Parents'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nasri', 'people.person.profession', 'Singer'), ('Lady Gaga', 'music.artist.genre', 'Contemporary R&B'), ('Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('As Long as You Love Me', 'music.album.compositions', 'As Long as You Love Me'), ('Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'), ('The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'), ('Bigger', 'music.composition.composer', 'Frank Ocean'), ('Bigger', 'music.composition.recordings', 'Bigger'), ('Canadian', 'common.topic.notable_types', 'Ethnicity'), ('As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'), ('Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chef Tone', 'people.person.nationality', 'United States of America'), ('Whitney Houston', 'music.artist.genre', 'Dance music'), ('My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'), ('Avery', 'music.artist.label', 'The Island Def Jam Music Group'), ('Change Me', 'music.album.primary_release', 'Change Me'), ('Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('m.0w3gbtv', 'film.personal_film_appearance.film', 'Zendaya: Behind the Scenes'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'), ('That Should Be Me', 'music.composition.form', 'Song'), ('Never Say Never', 'music.album.compositions', 'Never Say Never'), ('m.09wsj7g', 'common.webpage.topic', 'Teen idol'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Justin Bieber'), ('#thatPOWER', 'music.album.releases', '#thatPOWER'), ('Ashley Tisdale', 'people.person.profession', 'Actor'), ('Sir Nolan', 'music.artist.genre', 'Rock music'), ('Beauty and a Beat (acoustic version)', 'music.recording.song', 'Beauty And A Beat'), ('Ellen DeGeneres', 'people.person.nationality', 'United States of America'), ('Sia Furler', 'people.person.profession', 'Singer-songwriter'), ('Usher', 'music.composer.compositions', 'First Dance'), ('m.0n1ykxp', 'award.award_honor.award', 'MTV Video Music Award for Artist to Watch'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Rockumentary'), ('Amerie', 'people.person.gender', 'Female'), ('Real Change: Artists for Education', 'film.film.personal_appearances', 'm.0y5th3r'), ('Mistletoe', 'music.album.primary_release', 'Mistletoe'), ('Beautiful and the Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('Baby', 'common.topic.notable_types', 'Musical Album'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'), ('PYD', 'common.topic.notable_types', 'Composition'), ('Ashlee Simpson', 'people.person.profession', 'Singer'), ('Pray', 'music.album.artist', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'), ('Trey Songz', 'music.artist.genre', 'Contemporary R&B'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Geri Halliwell', 'people.person.profession', 'Model'), ('iJustine', 'people.person.gender', 'Female'), ('Nelly Furtado', 'people.person.gender', 'Female'), ('Trey Songz', 'people.person.nationality', 'United States of America'), ('m.0ng_vkd', 'film.personal_film_appearance.film', 'A Michael Bublé Christmas'), (\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Ludacris', 'music.composer.compositions', 'Baby'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'), ('Beyoncé Knowles', 'people.person.profession', 'Record producer'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'), ('Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'), ('CL', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Madonna', 'people.person.profession', 'Record producer'), ('m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'), ('1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'), ('PowerHitz', 'broadcast.content.artist', 'M.I.A.'), ('As Long as You Love Me (acoustic version)', 'music.recording.song', 'As Long as You Love Me'), ('Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'), ('Blu Cantrell', 'people.person.gender', 'Female'), ('Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'), ('Rob Thomas', 'people.person.gender', 'Male'), ('Singer', 'people.profession.specializations', 'Piano Singer'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'), ('NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'), ('m.0hvlt03', 'film.film_film_distributor_relationship.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'), ('Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_srv2b'), ('Terence Dudley', 'people.person.profession', 'Musician'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0101fszs', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Official website'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jenna Andrews'), ('FLOW 103', 'broadcast.content.artist', 'Cherish'), ('Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'), ('Next to You', 'music.recording.song', 'Next to You'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'), ('Ray J', 'people.person.nationality', 'United States of America'), ('Usher', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Madonna', 'influence.influence_node.influenced', 'Whitney Houston'), ('m.0w3gbtv', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Montell Jordan', 'music.artist.genre', 'Hip hop music'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Fabolous', 'broadcast.artist.content', 'PowerHitz'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jay-Z', 'common.topic.notable_types', 'Musical Artist'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Max Martin', 'freebase.valuenotation.has_value', 'Parents'), ('Record producer', 'common.topic.webpage', 'm.09ygb05'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'), ('m.0gbm3fj', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Iggy Azalea', 'music.artist.genre', 'Synthpop'), ('Tricky Stewart', 'common.topic.notable_types', 'Record Producer'), ('As Long As You Love Me (Ferry Corsten club dub)', 'common.topic.notable_types', 'Musical Recording'), ('#thatPOWER', 'music.album.album_content_type', 'Studio album'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Katy Perry', 'music.artist.genre', 'Electronic dance music'), ('Kid Cudi', 'people.person.profession', 'Record producer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'), ('m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Jaden Smith', 'people.person.profession', 'Dancer'), ('m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'), ('Keyshia Cole', 'people.person.profession', 'Record producer'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98y7'), ('Person', 'type.type.properties', 'Spouse (or domestic partner)'), ('Fall Out Boy', 'music.artist.origin', 'Chicago'), ('Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'), ('Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'), ('FLOW 103', 'broadcast.content.artist', '50 Cent'), ('Jordin Sparks', 'music.artist.genre', 'Dance-pop'), ('L.A. Reid', 'music.producer.releases_produced', 'My World'), ('L.A. Reid', 'people.person.gender', 'Male'), ('Jessie J', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'), ('1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('London', 'location.location.containedby', 'Ontario'), ('Justin Bieber: Never Say Never', 'film.film.film_set_decoration_by', 'Lia Roldan'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chris Brown', 'music.composer.compositions', 'Next to You'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'), ('Children', 'type.property.schema', 'Person'), ('Change Me', 'music.album.releases', 'Change Me'), ('RedOne', 'music.artist.label', 'Island Records'), ('School Gyrls', 'film.film.starring', 'm.0jztshx'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), ('m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Teen Choice Award for Choice Twitter Personality', 'award.award_category.winners', 'm.0wjhc6c'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'), ('m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Will Smith'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Lupe Fiasco'), ('Hikaru Utada', 'music.artist.label', 'Island Records'), ('Dr. Dre', 'people.person.profession', 'Record producer'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'music.composer.compositions', 'Change Me'), ('Right Here', 'common.topic.notable_types', 'Composition'), ('Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Height'), ('#Thatpower', 'music.recording.artist', 'Will i Am'), ('Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'), ('m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'), ('Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('m.0v90skf', 'award.award_honor.award', 'Billboard Music Award for Top Male Artist'), ('Ludacris', 'people.person.profession', 'Actor'), ('Heartbreaker', 'music.album.genre', 'Pop music'), ('Cameo appearance', 'tv.special_tv_performance_type.episode_performances', 'm.0v1lwt2'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teen idol', 'common.topic.webpage', 'm.0b47zvy'), ('1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'), ('Model', 'base.lightweight.profession.similar_professions', 'Actor'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'), ('Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'One Chance'), ('Never Let You Go', 'common.topic.notable_types', 'Composition'), ('Live My Life', 'common.topic.article', 'm.0j4453y'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'), ('Roller Coaster', 'award.award_nominated_work.award_nominations', 'm.0_x4zg3'), ('Chris Brown', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'), ('Lupe Fiasco', 'music.artist.genre', 'Hip hop music'), ('Teen pop', 'common.topic.article', 'm.02ny8z'), ('PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'), ('Iggy Azalea', 'people.person.gender', 'Female'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Adrienne Bailon', 'people.person.profession', 'Dancer'), ('Hip hop music', 'broadcast.genre.content', '181-beat'), ('m.0sgk_cw', 'award.award_honor.award', \"Kids' Choice Award for Favorite Song\"), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'), ('Iggy Azalea', 'music.artist.genre', 'Electronic dance music'), ('MTV Video Music Brazil Award for Best International Artist', 'award.award_category.winners', 'm.0yrhhqv'), ('Mariah Carey', 'music.artist.label', 'Island Records'), ('Music', 'common.topic.subject_of', 'POPPMusic.net'), ('Camagüey', 'common.topic.notable_types', 'City/Town/Village'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Favorite Girl', 'music.album.artist', 'Justin Bieber'), ('m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'), ('Britney Spears', 'people.person.profession', 'Singer'), ('Die in Your Arms', 'music.recording.song', 'Die in Your Arms'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'), ('Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Rock music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Chef Tone', 'music.artist.genre', 'Hip hop music'), ('Rudolph Isley', 'people.person.gender', 'Male'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Barry Weiss'), ('Beauty and a Beat (Bisbetic Instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0z1scxk'), ('Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'), ('Will Smith', 'people.person.profession', 'Actor'), ('Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Fabian', 'people.person.gender', 'Male'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'), ('Somebody to Love (remix)', 'music.album.primary_release', 'Somebody to Love (remix)'), ('HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'), ('Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Height'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'), ('Spouse', 'type.property.expected_type', 'Person'), ('m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'), ('Baby', 'music.recording.artist', 'Ludacris'), ('Rudolph Valentino', 'people.person.nationality', 'United States of America'), ('Hit-Boy', 'music.artist.genre', 'Hip hop music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPower', 'music.composition.recordings', '#Thatpower'), (\"Justin Bieber's Believe\", 'base.schemastaging.context_name.pronunciation', 'm.011h9_22'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'), ('Miley Cyrus', 'people.person.profession', 'Musician'), ('Justin Timberlake', 'people.person.gender', 'Male'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('m.0vp8rhw', 'music.recording_contribution.album', 'Beauty and a Beat (Remixes)'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'), ('Katy Perry: Part of Me', 'common.topic.notable_types', 'Award-Winning Work'), ('m.0jsmvv5', 'film.film_regional_release_date.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'), ('My Worlds: The Collection', 'music.album.release_type', 'Album'), ('All Around The World (featuring Ludacris)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'), ('1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'), ('m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'), ('Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'), ('School Boy Records', 'music.record_label.artist', 'Madison Beer'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'), ('Musical Album', 'freebase.type_hints.included_types', 'Topic'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien dubstep mix)'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Fergie', 'people.person.profession', 'Actor'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Stuart Ford', 'people.person.profession', 'Film Producer'), ('Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'), ('Zac Efron', 'people.person.gender', 'Male'), ('P!nk', 'music.artist.genre', 'Rock music'), ('R. Kelly', 'people.person.profession', 'Film Producer'), ('Gender', 'type.property.schema', 'Person'), ('Adam Messinger', 'music.artist.genre', 'Rhythm and blues'), ('Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'), ('Right Here', 'common.topic.notable_for', 'g.12h31mb_7'), ('JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Jessie J', 'influence.influence_node.influenced', 'Yves Bole'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Slick Rick'), ('Amerie', 'music.artist.genre', 'Rock music'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0pbzq13', 'film.performance.special_performance_type', 'Cameo appearance'), ('Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Height', 'type.property.unit', 'Meter'), ('Iggy Azalea', 'people.person.profession', 'Model'), ('NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'), ('Ray J', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Yves Bole', 'base.svocab.music_artist.genre', 'Pop'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Boyfriend (acoustic version)', 'music.recording.canonical_version', 'Boyfriend'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Believe Tour', 'music.concert_tour.album_or_release_supporting', 'Believe'), ('m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Believe Acoustic', 'music.album.release_type', 'Album'), ('Diplo', 'freebase.valuenotation.has_value', 'Height'), ('Hikaru Utada', 'music.artist.genre', 'Synthpop'), ('Roller Coaster', 'music.composition.composer', 'Julian Swirsky'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.song', 'As Long as You Love Me'), ('Elvis Presley', 'music.artist.genre', 'Pop music'), ('Lady Gaga', 'music.artist.genre', 'Pop music'), ('FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'), ('Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'), ('Usher', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Kelis', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0gbm3b7', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Twista', 'broadcast.artist.content', '.977 The Hits Channel'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'), ('All That Matters', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Nasri', 'music.artist.genre', 'Reggae'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'), ('m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'), ('Bad 25', 'film.film.genre', 'Documentary film'), ('Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'), ('Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me'), ('Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Geri Halliwell', 'people.person.profession', 'Musician'), ('Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'), ('Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Coldplay', 'music.artist.genre', 'Rock music'), ('Kevin Risto', 'people.person.profession', 'Record producer'), ('Whitney Houston', 'people.person.profession', 'Model'), ('Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'), ('Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'), ('181-beat', 'broadcast.content.artist', 'Cassie Ventura'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Disney Parks Christmas Day Parade', 'common.topic.notable_types', 'Award-Winning Work'), ('Ray J', 'people.person.profession', 'Artist'), ('Avril Lavigne', 'people.person.profession', 'Singer-songwriter'), ('American Music Award for Favorite Pop/Rock Male Artist', 'award.award_category.winners', 'm.0ndc0sf'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Music', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Christina Aguilera', 'music.artist.genre', 'Electronic music'), ('PowerHitz', 'broadcast.content.artist', 'Outkast'), ('U Smile', 'music.music_video.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.genre', 'Rock music'), ('Sean Kingston', 'music.artist.genre', 'Hip hop music'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Haley James Scott', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('Kylie Minogue', 'music.artist.genre', 'Rock music'), ('Chris Jasper', 'people.person.nationality', 'United States of America'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My Worlds: The Collection', 'music.album.album_content_type', 'Compilation album'), ('Lolly', 'music.album.releases', 'Lolly'), ('Toby Gad', 'common.topic.notable_types', 'Record Producer'), ('That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'), ('1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'), ('m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'), ('1Club.FM: Power', 'broadcast.content.artist', 'DMX'), ('Ja Rule', 'people.person.profession', 'Singer'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'), ('Die in Your Arms', 'award.award_nominated_work.award_nominations', 'm.0z85qxq'), ('Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'), ('m.012nv5gz', 'people.place_lived.location', 'Camagüey'), ('Kuk Harrell', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft5f'), ('Somebody to Love (J Stax remix)', 'music.recording.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Allison Kaye Scarinzi'), ('Adam Messinger', 'people.person.nationality', 'Canada'), ('Nasri', 'music.artist.genre', 'Pop music'), ('#thatPower', 'music.recording.featured_artists', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'), ('C1', 'common.topic.notable_types', 'Musical Artist'), ('.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'), ('School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'), ('Country', 'freebase.type_profile.strict_included_types', 'Topic'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.05sp405', 'organization.organization_relationship.child', 'Island Records'), ('Savan Kotecha', 'people.person.profession', 'Record producer'), ('Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'), ('m.0b47zvy', 'common.webpage.topic', 'Teen idol'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Teen Choice Award for Choice Summer Music Star: Male', 'award.award_category.winners', 'm.0yrjynf'), ('Juicy J', 'people.person.profession', 'Actor'), ('m.0yqflrk', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('1Club.FM: Power', 'broadcast.content.artist', 'Eminem'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('m.04q65lb', 'organization.organization_relationship.child', 'The Island Def Jam Music Group'), ('Big Sean', 'people.person.nationality', 'United States of America'), ('Beyoncé Knowles', 'people.person.profession', 'Film Producer'), ('R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'), ('1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'), ('Geri Halliwell', 'people.person.profession', 'Actor'), ('Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('My World', 'music.album.artist', 'Justin Bieber'), ('Don Henley', 'people.person.gender', 'Male'), ('HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'), ('Musician', 'people.profession.specializations', 'Singer'), ('Die in Your Arms', 'music.recording.canonical_version', 'Die in Your Arms'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0njvs9s', 'award.award_honor.award', 'CMT Music Award: Collaborative Video of the Year'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber', 'music.artist.album', 'Turn to You (Mother’s Day Dedication)'), ('Ludacris', 'music.artist.contribution', 'm.0vmyv4w'), ('Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'), ('City/Town/Village', 'freebase.type_profile.strict_included_types', 'Topic'), ('Recovery', 'common.topic.notable_types', 'Musical Recording'), ('Dancer', 'common.topic.notable_types', 'Profession'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('Terence Dudley', 'people.person.gender', 'Male'), ('Baby', 'music.composition.recordings', 'Polka Face'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mr. Sosa & The Yayo'), ('Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Rihanna', 'music.artist.genre', 'Dance music'), ('justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'), ('SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Gender', 'type.property.expected_type', 'Gender'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101fvbf', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0yrhrwc', 'award.award_honor.ceremony', '2011 MTV Video Music Aid Japan'), ('MTV Europe Music Award for Best North American Act', 'award.award_category.winners', 'm.0yrhmll'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'), ('m.0101ft1d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'), ('Ciara', 'people.person.gender', 'Female'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'), ('Britney Spears', 'music.artist.genre', 'Synthpop'), ('Thought of You', 'music.recording.artist', 'Justin Bieber'), ('m.0jzrrqs', 'location.mailing_address.country', 'United States of America'), ('Justin Bieber', 'internet.blogger.blog', 'justinbieber'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Toby Gad', 'people.person.nationality', 'United States of America'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Hot Wired Radio', 'broadcast.content.genre', 'Rock music'), ('m.0z8qqh5', 'award.award_nomination.ceremony', '2010 Teen Choice Awards'), ('50 Cent', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Die in Your Arms', 'common.topic.notable_types', 'Composition'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Height'), ('Change Me', 'award.award_nominated_work.award_nominations', 'm.0_w1gn3'), ('Justin Timberlake', 'broadcast.artist.content', 'Hot Wired Radio'), ('All Around The World', 'common.topic.article', 'm.09c4dl'), ('Europop', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('Country', 'freebase.type_profile.strict_included_types', 'Location'), ('Jason Mraz', 'people.person.profession', 'Singer-songwriter'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rune Reilly Kølsch'), ('Turn to You (Mother’s Day Dedication)', 'common.topic.notable_types', 'Musical Album'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rudolph Isley', 'music.artist.genre', 'Rock music'), ('Singer', 'people.profession.specializations', 'Chansonnier'), ('m.0101ftmm', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Kelis', 'music.artist.genre', 'Electronic dance music'), ('Somebody to Love', 'common.topic.article', 'm.0bmc2qq'), ('Bigger', 'music.composition.lyricist', 'Justin Bieber'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Boyfriend', 'music.composition.composer', 'Matthew Musto'), ('m.0gbm3cx', 'film.personal_film_appearance.person', 'Usher'), ('Roller Coaster', 'common.topic.notable_types', 'Composition'), ('Avril Lavigne', 'music.artist.genre', 'Rock music'), ('The Pussycat Dolls', 'music.artist.genre', 'Rhythm and blues'), ('Right Here', 'common.topic.notable_types', 'Musical Album'), ('Johntá Austin', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'influence.influence_node.influenced', 'Yves Bole'), ('Britney Spears', 'broadcast.artist.content', '1Club.FM: Power'), ('Kelly Clarkson', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('#thatPOWER', 'music.recording.tracks', '#thatpower'), ('Yves Bole', 'base.musicpf.artist.genre', 'Pop music'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('PowerHitz', 'broadcast.content.artist', 'Trick Daddy'), ('Haley James Scott', 'people.person.gender', 'Female'), ('Justin Bieber', 'music.artist.album', 'Believe Acoustic'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'), ('Deborah Lurie', 'people.person.gender', 'Female'), ('Jordan Francis', 'people.person.gender', 'Male'), ('Big R Radio - The Hawk', 'common.topic.article', 'm.03hs0cl'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0wjgrzc'), ('Children', 'rdf-schema#domain', 'Person'), ('Ciara', 'people.person.profession', 'Record producer'), ('Linkin Park', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('All That Matters', 'award.award_nominated_work.award_nominations', 'm.0_vw6nn'), ('First Dance', 'music.composition.composer', 'Alexander Parhm, Jr.'), ('Blackstreet', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Leif Garrett', 'music.artist.genre', 'Pop music'), ('Duffy', 'people.person.gender', 'Female'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Official website'), ('The Island Def Jam Music Group', 'freebase.valuenotation.has_value', 'Founders'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0f0dwc4'), ('HitzRadio.com', 'broadcast.content.artist', 'AnnaGrace'), ('m.0sxhgqj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Live My Life', 'music.composition.recordings', 'Live My Life (Jaywalker remix)'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('Christina Milian', 'people.person.nationality', 'United States of America'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Eminem', 'broadcast.artist.content', '.977 The Hits Channel'), ('Ciara', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Dapo Torimiro', 'people.person.profession', 'Musician'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Gwen Stefani', 'music.artist.genre', 'Rhythm and blues'), ('Love Never Felt So Good', 'music.album.release_type', 'Single'), ('Never Say Never', 'music.recording.canonical_version', 'Never Say Never'), ('CL', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sean Combs', 'people.person.profession', 'Musician'), ('Justin Bieber', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Green Day', 'common.topic.notable_types', 'Musical Artist'), ('Anastacia', 'people.person.profession', 'Actor'), ('As Long as You Love Me', 'music.album.releases', 'As Long as You Love Me'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.episode', 'Tina Fey / Justin Bieber'), ('Roller Coaster', 'common.topic.notable_types', 'Musical Album'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Remix)'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.012m1vz9', 'music.group_membership.role', 'Record producer'), ('Lil Jon', 'common.topic.notable_types', 'Musical Artist'), ('Nelly Furtado', 'music.artist.genre', 'Dance music'), ('FLOW 103', 'broadcast.content.artist', 'Justin Timberlake'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Nikki Williams'), ('Selena Gomez', 'music.artist.genre', 'Dance-pop'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('m.0gbcs16', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0y5th3r', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Live My Life', 'music.album.featured_artists', 'Justin Bieber'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('PowerHitz', 'broadcast.content.artist', 'Chingy'), ('Jessie J', 'people.person.profession', 'Musician'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Hip hop music'), ('Lady Gaga', 'broadcast.artist.content', 'HitzRadio.com'), (\"Destiny's Child\", 'music.artist.genre', 'Rhythm and blues'), ('Elvis Presley', 'people.person.languages', 'English Language'), ('Beautiful', 'music.composition.lyricist', 'Alex Lambert'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Redfoo', 'music.artist.genre', 'Pop music'), ('Beyoncé Knowles', 'broadcast.artist.content', '181-beat'), ('David Cassidy', 'people.person.languages', 'English Language'), ('Jason Mraz', 'broadcast.artist.content', 'Sunshine Radio'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'people.person.gender', 'Female'), ('m.012bm2v1', 'celebrities.friendship.friend', 'iJustine'), ('The Island Def Jam Music Group', 'organization.organization.leadership', 'm.0ncpx3v'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftk6'), ('Never Say Never', 'music.album.contributor', 'm.0vp7qr5'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Justin Bieber: Never Say Never', 'film.film.release_date_s', 'm.0jsmvv5'), ('One Time', 'common.topic.article', 'm.0k20sjb'), ('My World', 'music.album.releases', 'My World'), ('Little Bird', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Aaliyah', 'music.artist.genre', 'Rock music'), ('Benny Blanco', 'music.artist.genre', 'Hip hop music'), ('Madonna', 'music.artist.genre', 'Rock music'), ('Bad Day', 'music.composition.recordings', 'Bad Day'), ('Demi Lovato', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('My Worlds Acoustic', 'music.album.releases', 'My Worlds Acoustic'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Leona Lewis'), ('Pearl Jam', 'influence.influence_node.influenced', 'Kings of Leon'), ('Britney Spears', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('My Worlds', 'music.album.album_content_type', 'Compilation album'), ('First Dance', 'music.composition.recordings', 'First Dance'), ('#thatPOWER', 'music.album.primary_release', '#thatPOWER'), ('Caitlin Beadles', 'base.popstra.celebrity.dated', 'm.0gxnp0c'), ('Beauty and a Beat (Wideboys Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Janet Jackson', 'people.person.profession', 'Dancer'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO RNB Mix'), ('HitzRadio.com', 'broadcast.content.genre', 'Rap music'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Rap music'), ('m.0yrk18w', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Next to You', 'common.topic.notable_for', 'g.126smxx6r'), ('Contemporary R&B', 'music.genre.recordings', 'Daforce Is Wit Me'), ('m.0101fvj9', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('m.0j_tkcq', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Jordin Sparks', 'people.person.profession', 'Singer'), ('Frank Ocean', 'music.composer.compositions', 'Bigger'), ('2013 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('Madonna', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('m.0gfpbq5', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('Jordan Francis', 'people.person.profession', 'Singer-songwriter'), ('Green Day', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Timbaland', 'music.artist.genre', 'Rap music'), ('2013 MTV Europe Music Awards', 'time.event.people_involved', 'Redfoo'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Dance music'), ('Jane Lipsitz', 'film.producer.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Garrett Grant'), ('Disney Parks Christmas Day Parade', 'film.film.personal_appearances', 'm.0v_714c'), ('181-beat', 'broadcast.content.artist', 'Akon'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sonique'), ('Hip hop music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('4th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0y_g556'), ('Whitney Houston', 'music.artist.genre', 'Pop music'), ('Janet Jackson', 'music.artist.genre', 'Contemporary R&B'), ('Alanis Morissette', 'music.artist.genre', 'Rock music'), ('Kylie Minogue', 'music.artist.genre', 'Dance music'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Radio Mix)'), ('Haley James Scott', 'fictional_universe.fictional_character.gender', 'Female'), ('Mariah Carey', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0njw444', 'award.award_honor.ceremony', '2011 MTV Europe Music Awards'), ('Teyana', 'music.artist.genre', 'Contemporary R&B'), ('Kylie Minogue', 'people.person.languages', 'English Language'), ('Will i Am', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft1r'), ('HitzRadio.com', 'broadcast.content.artist', 'Lil Jon'), ('m.0njhyh_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Christina Aguilera', 'people.person.profession', 'Dancer'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('One Time', 'music.album.release_type', 'Single'), ('Taylor Swift', 'people.person.profession', 'Singer-songwriter'), ('Iggy Azalea', 'people.person.profession', 'Singer'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock Remix)'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njdns_'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Top 40'), ('Chicago', 'base.biblioness.bibs_topic.is_really', 'Chicago'), ('FLOW 103', 'broadcast.content.genre', 'Pop music'), ('Lolly', 'music.single.versions', 'Lolly'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Actor'), ('Justin Bieber', 'people.person.parents', 'Pattie Mallette'), ('Soulja Boy', 'people.person.nationality', 'United States of America'), ('Next to You', 'music.recording.artist', 'Justin Bieber'), ('Journals', 'common.topic.notable_for', 'g.1ypm_b95r'), ('Alicia Keys', 'music.artist.genre', 'Electronic music'), ('All Bad', 'music.composition.composer', 'Ryan Toby'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ashlee Simpson', 'people.person.gender', 'Female'), ('The Island Def Jam Music Group', 'common.topic.notable_types', 'Record label'), ('Snoop Dogg', 'music.artist.genre', 'Contemporary R&B'), ('Katy Perry', 'film.producer.film', 'Katy Perry: Part of Me'), ('Rob Thomas', 'people.person.profession', 'Singer-songwriter'), ('L.A. Reid', 'film.producer.film', 'School Gyrls'), ('181-beat', 'broadcast.content.artist', 'Mario Winans'), ('Shaffer Smith', 'broadcast.artist.content', 'PowerHitz'), ('Juvenile', 'music.artist.genre', 'Hip hop music'), ('m.0v1lwt2', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Eminem', 'people.person.profession', 'Film Producer'), ('RedOne', 'people.person.gender', 'Male'), ('50 Cent', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98y7'), ('Victoria Justice', 'people.person.nationality', 'United States of America'), ('Kanye West', 'people.person.profession', 'Record producer'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0yr9tjb'), ('A Star Was Born: Justin Bieber', 'film.film.personal_appearances', 'm.0h4yhbb'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y803nt'), ('Justin Bieber', 'book.author.works_written', 'Justin Bieber: Just Getting Started'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Cascada'), ('.977 The Hits Channel', 'broadcast.content.artist', 'P!nk'), ('Yves Bole', 'people.person.parents', 'Maribel Alonso Diaz'), ('Ciara', 'broadcast.artist.content', 'Hot 108 Jamz'), ('My World 2.0', 'music.album.genre', 'Pop music'), ('Never Say Never: The Remixes', 'music.album.genre', 'Pop music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Antebellum'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Height'), ('Yves Bole', 'common.topic.notable_for', 'g.11b75sb023'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y7y53r'), ('m.0y_g42w', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.form', 'Song'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Switchfoot'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvjw'), ('Big R Radio - The Hawk', 'broadcast.content.broadcast', 'Big R Radio - The Hawk - 128kbps Stream'), ('m.0pcr28j', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Marvin Isley', 'people.person.profession', 'Singer'), ('DMX', 'people.person.profession', 'Actor'), ('A Michael Bublé Christmas', 'film.film.language', 'English Language'), ('Musical Artist', 'freebase.type_hints.included_types', 'Topic'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'OneRepublic'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life (Jaywalker remix)'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Timbaland', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.artist.album', 'As Long as You Love Me'), ('First Dance', 'common.topic.image', 'Justin Bieber in concert crop'), ('Rodney Jerkins', 'music.artist.genre', 'Dance music'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Height'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Shaffer Smith'), ('Justin Bieber', 'music.composer.compositions', 'Confident'), ('Nasri', 'music.artist.genre', 'Contemporary R&B'), ('All Around The World (featuring Ludacris)', 'music.recording.canonical_version', 'All Around the World'), ('Eminem', 'people.person.nationality', 'United States of America'), ('m.0sxhq3d', 'award.award_nomination.ceremony', 'Juno Awards of 2013'), ('m.0t_l7mp', 'award.award_nomination.nominated_for', 'Believe'), ('Beyoncé Knowles', 'music.artist.genre', 'Hip hop music'), ('Die in Your Arms', 'common.topic.article', 'm.0jwx2__'), ('Johntá Austin', 'music.composer.compositions', 'Never Let You Go'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfmm45'), ('Height', 'type.property.schema', 'Person'), ('m.0ng_k21', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0ndc0sf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'music.artist.album', 'Right Here'), ('Record producer', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('Fergie', 'people.person.nationality', 'United States of America'), ('Beautiful', 'music.recording.song', 'Beautiful'), ('Jason Mraz', 'broadcast.artist.content', 'radioIO Todays POP'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Wayne Wonder'), ('Under the Mistletoe', 'music.album.genre', 'Reggae'), ('Boyfriend (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Christina Milian', 'music.artist.genre', 'Pop music'), ('m.0jztshx', 'film.performance.film', 'School Gyrls'), ('#Thatpower', 'common.topic.notable_types', 'Musical Recording'), ('Dr. Dre', 'people.person.profession', 'Film Producer'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0z8qx3w'), ('Johntá Austin', 'common.topic.notable_types', 'Musical Artist'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Amerie'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjdc'), ('Sia Furler', 'music.artist.genre', 'Dance-pop'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Fabolous'), ('181-beat', 'broadcast.content.artist', 'Shaffer Smith'), ('Beauty And A Beat', 'music.composition.composer', 'Max Martin'), ('Selena Gomez', 'music.artist.genre', 'Rhythm and blues'), ('Beauty and a Beat', 'music.single.versions', 'Beautiful and the Beat'), ('School Gyrls', 'common.topic.notable_types', 'Film'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftm6'), (\"Justin Bieber's Believe\", 'film.film.edited_by', 'Avi Youabian'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Parents'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Profession'), ('KooL CrAzE', 'common.topic.notable_types', 'Musical Artist'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sir Nolan', 'people.person.profession', 'Musician'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.song', 'As Long as You Love Me'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Instrumental)'), ('Iggy Azalea', 'music.artist.genre', 'Rhythm and blues'), ('Nicki Minaj', 'music.featured_artist.albums', 'Beauty and a Beat'), ('m.04kjw8n', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('m.0njdns_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Ja Rule', 'people.person.nationality', 'United States of America'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ice Cube'), ('Teen idol', 'base.icons.icon_genre.icons', 'Paul Anka'), ('U Smile', 'award.award_winning_work.awards_won', 'm.0n4rmg7'), ('Hot Wired Radio', 'broadcast.content.artist', 'Ciara'), ('Roller Coaster', 'music.composition.recordings', 'Roller Coaster'), ('Barry Weiss', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Fray'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('m.0y803nt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Shaggy', 'people.person.gender', 'Male'), ('Kelis', 'broadcast.artist.content', 'radioIO Todays POP'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Sia Furler', 'music.artist.genre', 'Electronic dance music'), ('Juvenile', 'broadcast.artist.content', '.977 The Hits Channel'), ('Rudolph Isley', 'music.artist.genre', 'Rhythm and blues'), ('Ja Rule', 'music.artist.genre', 'Contemporary R&B'), ('m.0z1jn32', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Santana', 'broadcast.artist.content', 'Sunshine Radio'), ('Billboard Music Milestone Award', 'award.award_category.winners', 'm.0vb6hhj'), ('Selena Gomez', 'people.person.profession', 'Singer'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My World', 'music.album.genre', 'Teen pop'), ('Yves Bole', 'people.person.languages', 'Italian Language'), ('PowerHitz', 'broadcast.content.artist', 'Lady Gaga'), ('m.0gxnp02', 'base.popstra.dated.participant', 'Selena Gomez'), ('Contemporary R&B', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('Boyfriend', 'music.recording.canonical_version', 'Boyfriend'), ('Rudolph Valentino', 'base.icons.icon.icon_genre', 'Teen idol'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0ywsnc9'), ('m.0z1scxk', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Sash!'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Leona Lewis'), ('m.0101ft3d', 'film.personal_film_appearance.person', 'Ellen DeGeneres'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Lifehouse'), ('Willa Ford', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Elvis Presley'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mannie Fresh'), ('1Club.FM: Power', 'broadcast.content.artist', 'Will Smith'), ('Dapo Torimiro', 'music.artist.genre', 'Contemporary R&B'), ('Ja Rule', 'music.artist.genre', 'Pop music'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien Luvstep mix)'), ('Taylor Swift', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.release_date_s', 'm.0j593k1'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('All Around the World (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_98_q', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Alicia Keys', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnt3'), ('m.0_svs8m', 'award.award_nomination.nominated_for', 'justinbieber'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0ndc0sf', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Male Artist'), ('Justin Bieber: First Step 2 Forever', 'common.topic.notable_types', 'Written Work'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Ludacris', 'broadcast.artist.content', '.977 The Hits Channel'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('m.0zbg2kw', 'award.award_nomination.nominated_for', 'My World 2.0'), ('PYD', 'music.composition.composer', 'R. Kelly'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Gender'), ('HitzRadio.com', 'broadcast.content.artist', 'Mims'), ('Wait For a Minute', 'music.recording.artist', 'Justin Bieber'), ('Place of birth', 'type.property.schema', 'Person'), ('Beyoncé Knowles', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: 80s (Pop)', 'broadcast.content.producer', '1Club.FM'), ('m.0gctytd', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Akon', 'people.person.gender', 'Male'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Katy Perry', 'people.person.profession', 'Musician'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('U Smile', 'common.topic.notable_types', 'Award-Winning Work'), ('Mannie Fresh', 'people.person.gender', 'Male'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('Never Say Never: The Remixes', 'common.topic.notable_types', 'Musical Album'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rodney Jerkins', 'freebase.valuenotation.has_value', 'Parents'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg8bc1'), ('Mary J. Blige', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'World music'), ('Donna Summer', 'people.person.nationality', 'United States of America'), ('Pop music', 'music.genre.parent_genre', 'Rock music'), ('Johntá Austin', 'music.artist.genre', 'Pop music'), ('Leonardo DiCaprio', 'people.person.gender', 'Male'), ('m.0j2gfc1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Hot Wired Radio', 'broadcast.content.artist', 'Will i Am'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Journals', 'music.album.releases', 'Journals'), ('Rihanna', 'broadcast.artist.content', '.977 The Hits Channel'), ('Terence Dudley', 'music.artist.genre', 'Hip hop music'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Hip hop music', 'common.topic.subjects', 'Brian Keith Kennedy'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'First Dance'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Island Records', 'music.record_label.releases', 'Believe'), ('m.0z8qqh5', 'award.award_nomination.nominated_for', 'My World 2.0'), ('David Nicksay', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'music.artist.album', 'Wait For a Minute'), ('As Long As You Love Me (Audiobot edit)', 'common.topic.notable_types', 'Musical Recording'), ('Will i Am', 'people.person.profession', 'Film Producer'), ('Nyíregyháza', 'common.topic.notable_types', 'City/Town/Village'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv62'), ('HitzRadio.com', 'broadcast.content.artist', 'Jason Mraz'), ('The Notorious B.I.G.', 'people.person.profession', 'Artist'), ('Will i Am', 'people.person.profession', 'Musician'), ('Mariah Carey', 'music.artist.genre', 'Dance music'), ('Confident', 'music.composition.composer', 'Chance the Rapper'), ('m.0tkc3tj', 'award.award_nomination.nominated_for', 'My World'), ('Selena Gomez', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.composition.composer', 'Bilal Hajji'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All That Matters', 'music.composition.composer', 'Donovan Knight'), ('m.09ygb05', 'common.webpage.topic', 'Record producer'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Stratford', 'common.topic.notable_types', 'City/Town/Village'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.composer', 'Adam Messinger'), ('Ray J', 'music.artist.genre', 'Pop music'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Nasri', 'music.composer.compositions', 'Next to You'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6_81'), ('Die in Your Arms', 'music.album.releases', 'Die in Your Arms'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Panic! at the Disco'), ('#thatPower', 'music.composition.recordings', '#Thatpower'), ('m.0101fvc3', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Green Day'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Jayceon Terrell Taylor', 'music.artist.genre', 'Hip hop music'), ('m.0ndc3_1', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0tjyljn', 'award.award_nomination.ceremony', 'Juno Awards of 2011'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Weezer'), ('Jordan Pruitt', 'people.person.gender', 'Female'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot edit)'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'TLC'), ('Michael Jackson', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0z1scxk'), ('Mariah Carey', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Timberlake', 'broadcast.artist.content', '1.FM Top 40'), ('We Are the World 25 for Haiti', 'music.album.release_type', 'Single'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'common.topic.notable_types', 'Film'), ('Hikaru Utada', 'people.person.gender', 'Female'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)'), ('Justin Bieber', 'music.composer.compositions', 'Boyfriend'), ('Big R Radio - Top 40 Hits', 'common.topic.image', 'Big R radio'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juelz Santana', 'people.person.profession', 'Record producer'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Pop music', 'common.topic.subjects', 'Bleona'), ('Electronic dance music', 'music.genre.subgenre', 'Eurodance'), ('Beyoncé Knowles', 'broadcast.artist.content', '1.FM Top 40'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Sean Paul'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h5m2b'), ('Shaggy', 'people.person.nationality', 'United States of America'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0h4yhbb', 'film.personal_film_appearance.person', 'Justin Bieber'), ('m.0sgkrj4', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jay Cassidy', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.song', 'As Long as You Love Me'), ('Tupac Shakur', 'music.artist.genre', 'Hip hop music'), ('Ray J', 'people.person.gender', 'Male'), ('Justin Bieber', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Tricky Stewart', 'people.person.gender', 'Male'), ('m.0njvvj_', 'award.award_honor.ceremony', 'Juno Awards of 2012'), ('Eminem', 'broadcast.artist.content', 'Sunshine Radio'), ('Live My Life (Party Rock remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('Zendaya: Behind the Scenes', 'film.film.genre', 'Documentary film'), ('Hikaru Utada', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0vb6hhj'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvph'), ('Vanessa Hudgens', 'music.artist.genre', 'Dance music'), ('m.0j2gfc1', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Geri Halliwell', 'broadcast.artist.content', '1.FM Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074l61'), ('Christina Aguilera', 'broadcast.artist.content', 'radioIO Todays POP'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Boys Like Girls'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat'), ('Never Let You Go', 'music.composition.composer', 'Johntá Austin'), ('Johntá Austin', 'people.person.profession', 'Singer'), ('m.0yrk0mt', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('Live My Life', 'music.composition.composer', 'Martin Kierszenbaum'), ('Live My Life', 'music.composition.composer', 'RedOne'), ('Synthpop', 'music.genre.parent_genre', 'Pop music'), ('Janet Jackson', 'people.person.profession', 'Film Producer'), ('1.FM Top 40', 'broadcast.content.artist', 'Twista'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k3w'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvps'), ('Composition', 'freebase.type_hints.included_types', 'Topic'), ('radioIO Classic RNB', 'common.topic.image', 'RadioIO.png'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me (album version)'), ('Confident', 'common.topic.notable_types', 'Composition'), ('radioIO Todays POP', 'common.topic.article', 'm.03hs0q2'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'All Around The World'), ('1.FM Top 40', 'broadcast.content.genre', 'Classic hits'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('m.0gbm35z', 'film.film_cut.film', 'Justin Bieber: Never Say Never'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('justinbieber', 'internet.blog.language', 'English Language'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Aaliyah'), ('RedOne', 'music.composer.compositions', 'Live My Life'), ('Victoria Justice', 'people.person.profession', 'Musician'), ('Wait For a Minute', 'common.topic.notable_types', 'Musical Recording'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0y_g42w', 'award.award_nomination.ceremony', '4th Annual Shorty Awards'), ('Shorty Award for Music', 'award.award_category.winners', 'm.0z0tmyv'), ('United States of America', 'location.location.time_zones', 'Eastern Time Zone'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Radio Mix)'), ('m.0ywsnc9', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Somebody to Love', 'music.album.release_type', 'Single'), ('Yves Bole', 'influence.influence_node.influenced', 'Lady Gaga'), ('Ciara', 'people.person.profession', 'Dancer'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Official website'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'common.topic.notable_types', 'Musical Recording'), ('HitzRadio.com', 'broadcast.content.artist', 'Beyoncé Knowles'), ('As Long As You Love Me (Audien dubstep edit)', 'common.topic.notable_types', 'Musical Recording'), ('Savan Kotecha', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Journals', 'music.album.artist', 'Justin Bieber'), ('Change Me', 'music.composition.recordings', 'Change Me'), ('Whitney Houston', 'influence.influence_node.influenced', 'Brandy Norwood'), ('Don Henley', 'common.topic.notable_types', 'Musical Artist'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Place of birth'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('P!nk', 'music.artist.genre', 'Dance-pop'), ('m.0101fszb', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Jay-Z', 'music.artist.label', 'The Island Def Jam Music Group'), ('Demi Lovato', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.album', 'Beauty and a Beat'), ('1.FM Top 40', 'broadcast.content.artist', 'Mariah Carey'), ('m.0z85n_1', 'award.award_nomination.nominated_for', 'All Around The World'), ('Where Are Ü Now', 'common.topic.notable_types', 'Musical Recording'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Encore'), ('Jay-Z', 'people.person.profession', 'Record producer'), ('Max Martin', 'people.person.gender', 'Male'), ('Contemporary R&B', 'music.genre.parent_genre', 'Arabic pop music'), ('m.0n58kgb', 'award.award_nomination.award', 'American Music Award for Favorite Pop/Rock Album'), ('Heartbreaker', 'common.topic.notable_types', 'Composition'), ('R. Kelly', 'music.artist.genre', 'Dance-pop'), ('Verse Simmonds', 'freebase.valuenotation.has_value', 'Date of birth'), ('Terius Nash', 'music.artist.genre', 'Synthpop'), ('Estelle', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.012nh7h_', 'theater.theater_role.role', 'Singer'), ('Jaden Smith', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('PowerHitz', 'broadcast.content.genre', 'Urban contemporary'), ('Hot 108 Jamz', 'broadcast.content.artist', 'DMX'), ('Jordan Pruitt', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Dany Brillant', 'common.topic.notable_types', 'Musical Artist'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Fray'), ('Justin Bieber', 'music.composer.compositions', '#thatPower'), ('Michael Jackson', 'people.person.profession', 'Singer-songwriter'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Sir Mix-a-Lot', 'people.person.gender', 'Male'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Alicia Keys', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Channel One', 'broadcast.content.location', 'Chicago'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9j0d'), ('m.046c33q', 'common.webpage.category', 'Topic Webpage'), ('Baby', 'music.composition.recordings', 'Baby'), ('Believe Tour', 'common.topic.notable_types', 'Concert tour'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Singer', 'people.profession.specializations', 'Carnatic Singer'), ('William Orbit', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Baby', 'music.composition.recordings', 'Baby'), ('Hikaru Utada', 'people.person.nationality', 'United States of America'), ('Die in Your Arms', 'common.topic.notable_types', 'Musical Album'), ('Never Say Never', 'music.recording.featured_artists', 'Jaden Smith'), ('Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Record producer', 'common.topic.webpage', 'm.0b497pt'), ('m.012bm3j9', 'celebrities.friendship.friend', 'Justin Bieber'), ('Justin Bieber', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sean Combs'), ('m.0gbmntw', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011c0k9z'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Beyoncé Knowles', 'music.artist.genre', 'Rhythm and blues'), ('Sunshine Radio', 'common.topic.image', 'sunshine_radio.gif'), ('Keyshia Cole', 'music.artist.genre', 'Hip hop music'), ('Scottish Canadian', 'people.ethnicity.included_in_group', 'Canadian'), ('1.FM Top 40', 'common.topic.article', 'm.03hs08z'), ('P!nk', 'people.person.profession', 'Dancer'), ('L.A. Reid', 'music.artist.genre', 'Hip hop music'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_706c'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Flyleaf'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkyfg'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Urban contemporary'), ('Frank Ocean', 'people.person.gender', 'Male'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Three Days Grace'), ('Little Bird', 'common.topic.notable_for', 'g.1yg9546vy'), ('m.0101fvmb', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0nhfd4m', 'award.award_nomination.nominated_for', 'Believe'), ('Max Martin', 'music.artist.genre', 'Rock music'), ('Diplo', 'people.person.nationality', 'United States of America'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Terence Dudley'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Kelly Clarkson', 'broadcast.artist.content', 'Hot Wired Radio'), ('Rodney Jerkins', 'music.artist.genre', 'Contemporary R&B'), ('Runaway Love (remix)', 'common.topic.notable_for', 'g.1259h0ss3'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Change Me', 'common.topic.notable_types', 'Composition'), ('Trick Daddy', 'music.artist.genre', 'Hip hop music'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('m.010htdc2', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot remix)'), ('Lady Gaga', 'music.artist.genre', 'Eurodance'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Height'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Date of birth'), ('Donna Summer', 'people.person.languages', 'English Language'), ('United States of America', 'location.country.languages_spoken', 'Spanish Language'), ('Believe', 'award.award_winning_work.awards_won', 'm.0ndc259'), ('Big R Radio - Top 40 Hits', 'broadcast.content.broadcast', 'Big R Radio - Top 40 Hits - 128kbps Stream'), ('Spouse', 'rdf-schema#range', 'Person'), ('Nelly', 'people.person.nationality', 'United States of America'), ('Billboard Music Award for Top Pop Album', 'award.award_category.winners', 'm.0njhxzc'), ('Tupac Shakur', 'people.person.profession', 'Writer'), ('Billboard Music Award for Top New Artist', 'award.award_category.winners', 'm.0njhtjj'), ('m.0z0tmyv', 'award.award_honor.honored_for', 'justinbieber'), ('m.0wjgqck', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('181-beat', 'broadcast.content.artist', 'Nelly'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Madonna', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('The Isley Brothers', 'music.artist.genre', 'Pop music'), ('m.0gwhmhm', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0zg8bc1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Will Smith', 'broadcast.artist.content', '1.FM Top 40'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.episode', 'Meet The Cast & Results Show: Week 1'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('Diplo', 'music.artist.genre', 'Reggae'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'music.composition.recordings', 'Beautiful'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Nick Jonas', 'people.person.gender', 'Male'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot edit)'), ('m.0101ft3d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Snoop Dogg'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Children'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_sxby0'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('HitzRadio.com', 'common.topic.notable_for', 'g.1254y3j6d'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Timbaland'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gbm3fr', 'film.personal_film_appearance.person', 'Scooter Braun'), ('The Isley Brothers', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Fall Out Boy', 'broadcast.artist.content', 'radioIO Todays POP'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y_g556'), ('iJustine', 'people.person.nationality', 'United States of America'), ('Barry Weiss', 'people.person.profession', 'Music executive'), ('Justin Bieber', 'music.artist.album', 'Baby'), ('HitzRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.episode', 'True Concert'), ('2011 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('As Long as You Love Me (album version)', 'music.recording.artist', 'Justin Bieber'), ('Snoop Dogg', 'music.artist.genre', 'Reggae'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Christina Aguilera'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO Todays RNB'), ('Christina Milian', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0t_l7mp', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Everlast', 'people.person.profession', 'Actor'), ('Anastacia', 'music.artist.genre', 'Rock music'), ('Ginuwine', 'people.person.gender', 'Male'), ('Jason Mraz', 'music.artist.genre', 'Pop music'), ('Baby', 'common.topic.notable_for', 'g.12h2w5zfr'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01068tkg'), ('m.0v1lwt2', 'tv.tv_guest_role.episodes_appeared_in', 'The Fabulous Faker Boy'), ('Juvenile', 'common.topic.notable_types', 'Musical Artist'), ('DMX', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Christina Aguilera', 'broadcast.artist.content', 'Sunshine Radio'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk0mt'), ('R. Kelly', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0njw59_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Children'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vb6hhj', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92x'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.artist', 'Justin Bieber'), ('Yves Bole', 'people.person.places_lived', 'm.012nv5gz'), ('Wait For a Minute', 'music.recording.artist', 'Tyga'), ('Believe', 'music.album.genre', 'Dance-pop'), ('Redfoo', 'people.person.profession', 'Record producer'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0pc67ly', 'award.award_nomination.ceremony', '2011 MTV Movie Awards'), ('Justin Bieber', 'music.composer.compositions', 'Eenie Meenie'), ('Kylie Minogue', 'broadcast.artist.content', 'radioIO Todays POP'), ('Mason Levy', 'freebase.valuenotation.has_value', 'Place of birth'), ('Jordan Francis', 'freebase.valuenotation.has_value', 'Parents'), ('HitzRadio.com', 'broadcast.content.artist', 'NB Ridaz'), ('Justin Bieber', 'music.featured_artist.albums', 'Next to You'), ('m.0njw1tn', 'award.award_honor.award_winner', 'Justin Bieber'), ('My World 2.0', 'common.topic.article', 'm.09v5jd_'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Miley Cyrus'), ('The Island Def Jam Music Group', 'organization.organization.headquarters', 'm.0jzrrqs'), (\"2011 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkyfg'), ('Justin Bieber Pictures', 'common.resource.annotations', 'm.0bvmhvb'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v1d2xz', 'award.award_nomination.nominated_for', 'Boyfriend'), ('Contemporary R&B', 'broadcast.genre.content', '181-rnb'), ('Anastacia', 'people.person.profession', 'Musician'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Mannie Fresh', 'people.person.profession', 'Record producer'), ('m.0h4yhbb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('#thatPower', 'common.topic.notable_for', 'g.11b5m0k2ty'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Avery'), ('Miley Cyrus', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Jordin Sparks', 'people.person.gender', 'Female'), ('Bruce Dickinson', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0ywtfj6'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0yr9tjb', 'award.award_nomination.nominated_for', 'My World'), ('Timbaland', 'music.artist.genre', 'Synthpop'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('m.03hs4ny', 'common.webpage.category', 'Topic Webpage'), ('Soulja Boy', 'people.person.profession', 'Record producer'), ('Snoop Dogg', 'music.artist.genre', 'Rhythm and blues'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Children'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Never Say Never', 'music.album.releases', 'Never Say Never'), ('Ernie Isley', 'music.composer.compositions', 'Bad Day'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0zg8bc1'), ('Nicki Minaj', 'people.person.gender', 'Female'), ('Ginuwine', 'music.artist.genre', 'Dance music'), ('Bad 25', 'film.film.country', 'United States of America'), ('Heartbreaker', 'music.recording.artist', 'Justin Bieber'), ('Christina Aguilera', 'people.person.profession', 'Film Producer'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Musical Artist', 'freebase.type_profile.strict_included_types', 'Topic'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Isley Brothers'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Gender'), ('NME Awards 2012', 'award.award_ceremony.awards_presented', 'm.0z8755b'), ('2014 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.010lkp2z'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Love Never Felt So Good', 'music.composition.recorded_as_album', 'Love Never Felt So Good'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('First Dance', 'music.recording.featured_artists', 'Usher'), ('My World 2.0', 'common.topic.image', 'justin'), ('m.0gbm3f5', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', \"Destiny's Child\"), ('Under the Mistletoe', 'music.album.genre', 'Contemporary R&B'), ('1Club.FM: V101', 'broadcast.content.artist', 'Craig David'), ('Tommy Sands', 'people.person.profession', 'Singer'), ('Start Date', 'type.property.expected_type', 'Date/Time'), ('Jaxon Bieber', 'people.person.gender', 'Male'), ('Date of birth', 'rdf-schema#range', 'Date/Time'), ('Akon', 'broadcast.artist.content', '181-beat'), ('Savan Kotecha', 'freebase.valuenotation.has_value', 'Date of birth'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012bm3j9'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Record producer', 'common.topic.article', 'm.0dz40'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Van Halen', 'broadcast.artist.content', '1.FM Top 40'), ('m.0z2dr9y', 'award.award_nomination.award', 'Shorty Award for Music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Colbie Caillat'), ('Avril Lavigne', 'broadcast.artist.content', 'Big R Radio - The Hawk'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvl8'), ('Christina Aguilera', 'people.person.languages', 'English Language'), ('Profession', 'type.property.schema', 'Person'), ('Linkin Park', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Christina Milian', 'people.person.profession', 'Dancer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Chris Brown'), ('Everlast', 'people.person.nationality', 'United States of America'), ('Pras', 'people.person.profession', 'Singer'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Ashley Tisdale', 'people.person.gender', 'Female'), ('Soulja Boy', 'music.artist.genre', 'Dance-pop'), ('m.0y803nt', 'award.award_honor.award_winner', 'Justin Bieber'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0gj0bj6', 'tv.tv_guest_role.episodes_appeared_in', 'JLS and Justin Bieber @ R1BW'), ('Willa Ford', 'people.person.gender', 'Female'), ('Teen idol', 'common.topic.article', 'm.0mb9c'), ('Runaway Love (remix)', 'music.album.primary_release', 'Runaway Love (remix)'), ('Justin Bieber', 'music.artist.label', 'The Island Def Jam Music Group'), ('Max Martin', 'common.topic.notable_types', 'Record Producer'), ('m.0bldgxn', 'common.resource.annotations', 'm.07lkzw7'), ('Parents', 'rdf-schema#domain', 'Person'), ('Contemporary R&B', 'broadcast.genre.content', 'ProjectVIBE.net'), ('Foreign Remix', 'music.recording.artist', 'Trey Songz'), ('Playback Singer', 'common.topic.notable_types', 'Profession'), ('Justin Timberlake', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Justin Timberlake'), ('Kuk Harrell', 'music.artist.genre', 'Contemporary R&B'), ('Hikaru Utada', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.artist.album', 'Favorite Girl'), ('Baby (acoustic)', 'common.topic.notable_types', 'Musical Recording'), ('Jason Mraz', 'people.person.nationality', 'United States of America'), ('m.0pcr2qn', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0z0tgz6', 'award.award_nomination.award', 'Shorty Award for Music'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot edit)'), ('Ethnicity', 'freebase.type_profile.strict_included_types', 'Topic'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'R. Kelly'), ('Lil Wayne', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Baby', 'music.recording.song', 'Baby'), ('Musician', 'common.topic.subjects', 'Steve Blevins'), ('1.FM Top 40', 'broadcast.content.artist', 'OneRepublic'), ('Record producer', 'people.profession.specialization_of', 'Music executive'), ('Sunshine Radio', 'broadcast.content.broadcast', 'Sunshine Radio - 140kbps Stream'), ('DMX', 'broadcast.artist.content', '1Club.FM: Power'), ('J. Holiday', 'music.artist.label', 'The Island Def Jam Music Group'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw444'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty and a Beat (Remixes)', 'music.album.featured_artists', 'Nicki Minaj'), ('Beyoncé Knowles', 'broadcast.artist.content', 'PowerHitz'), ('m.0gxnp0c', 'base.popstra.dated.participant', 'Caitlin Beadles'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Pop music'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Gender'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Brad Haugen'), ('1.FM Top 40', 'broadcast.content.artist', 'Enrique Iglesias'), ('New Kids on the Block', 'common.topic.notable_types', 'Musical Artist'), ('FLOW 103', 'broadcast.content.artist', 'Ciara'), ('m.0_w3zrs', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Height'), ('Hot Wired Radio', 'broadcast.content.artist', 'Nickelback'), ('Alanis Morissette', 'people.person.profession', 'Actor'), ('Tyga', 'music.artist.genre', 'Contemporary R&B'), ('Justin Timberlake', 'people.person.languages', 'English Language'), ('Michael Jackson', 'music.artist.genre', 'Rock music'), ('Redfoo', 'music.featured_artist.albums', 'Live My Life'), ('1.FM Top 40', 'broadcast.content.genre', 'Hip hop music'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98_q'), ('Kuk Harrell', 'people.person.profession', 'Record producer'), ('Barry Weiss', 'music.artist.label', 'The Island Def Jam Music Group'), ('Island Records', 'common.topic.notable_types', 'Record label'), ('Frank Ocean', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ch!pz'), (\"Destiny's Child\", 'music.artist.genre', 'Dance-pop'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Redfoo', 'music.featured_artist.recordings', 'Little Bird'), ('J. Holiday', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_73g3'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kelis'), ('Akon', 'music.artist.genre', 'Contemporary R&B'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Contemporary R&B', 'broadcast.genre.content', '181-beat'), ('Usher', 'people.person.profession', 'Actor'), ('Everlast', 'people.person.profession', 'Singer'), ('Estelle', 'people.person.profession', 'Model'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Rock music'), ('LeAnn Rimes', 'broadcast.artist.content', '1.FM Top 40'), ('That Should Be Me', 'award.award_winning_work.awards_won', 'm.0njvs9s'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhtjj'), ('1Club.FM: V101', 'broadcast.content.artist', 'Janet Jackson'), ('Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Khalil', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('G (USA)', 'film.content_rating.country', 'United States of America'), ('m.0njwb81', 'award.award_honor.ceremony', '2010 MuchMusic Video Awards'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.song', 'Beauty And A Beat'), ('6 Tre G', 'music.artist.label', 'The Island Def Jam Music Group'), ('Shaun Cassidy', 'people.person.gender', 'Male'), ('m.0sgkw_d', 'award.award_honor.award', \"Kids' Choice Award for Biggest Slime Of The Night\"), ('Live My Life', 'music.composition.composer', 'JC Sindress'), ('Pray', 'music.composition.composer', 'Nasri'), ('Kelis', 'people.person.nationality', 'United States of America'), ('Trick Daddy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3bl'), ('1Club.FM: Power', 'broadcast.content.artist', 'Aaliyah'), ('m.0z1ndbl', 'award.award_nomination.nominated_for', 'justinbieber'), ('Justin Bieber', 'celebrities.celebrity.sexual_relationships', 'm.0d33gyj'), ('Christina Aguilera', 'music.artist.genre', 'Contemporary R&B'), ('Eenie Meenie', 'common.topic.article', 'm.0bbz4pf'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulSmoothJazz.com'), ('6 Tre G', 'common.topic.notable_types', 'Musical Artist'), ('m.0z8s562', 'award.award_honor.ceremony', 'NME Awards 2011'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0z0tgz6', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0gxnnwc', 'people.sibling_relationship.sibling', 'Jazmyn Bieber'), ('Benny Blanco', 'music.producer.releases_produced', 'My World'), ('Miley Cyrus', 'music.artist.genre', 'Dance music'), ('Ashlee Simpson', 'people.person.profession', 'Dancer'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Athan Grace', 'people.person.nationality', 'United States of America'), ('m.0wjgrzc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Boyfriend (Dada Life remix)', 'common.topic.notable_types', 'Musical Recording'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('1.FM Top 40', 'broadcast.content.artist', 'Javier Colon'), ('Jennifer Lopez', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0t4syfh', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.composer', 'Justin Bieber'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Justin Bieber', 'music.composer.compositions', 'Live My Life'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0vb6hhj'), ('Beauty And A Beat', 'music.composition.composer', 'Savan Kotecha'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Marvin Isley'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0zbf_4g'), ('m.0yr9tjb', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Roller Coaster', 'common.topic.notable_for', 'g.1yn5d20x3'), ('Justin Bieber', 'music.artist.album', 'Believe'), ('Twista', 'music.artist.origin', 'Chicago'), ('1.FM Top 40', 'broadcast.content.artist', 'Keane'), ('Gender', 'rdf-schema#range', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Jillian Twigger Moul'), ('Never Say Never: The Remixes', 'music.album.release_type', 'Album'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Snoop Dogg'), ('m.0njvvj_', 'award.award_honor.award', 'Juno Fan Choice Award'), (\"Destiny's Child\", 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cp'), ('HitzRadio.com', 'broadcast.content.artist', 'DMX'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Rihanna'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk0mt'), ('Somebody to Love (remix)', 'music.album.releases', 'Somebody to Love (remix)'), ('Shaffer Smith', 'people.person.profession', 'Actor'), ('First Dance', 'music.composition.composer', 'Jesse Wilson'), ('Sean Combs', 'people.person.profession', 'Singer'), ('Reed Smoot', 'people.person.nationality', 'United States of America'), ('Kings of Leon', 'influence.influence_node.influenced_by', 'Radiohead'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.featured_artists', 'Big Sean'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gj0bj6'), ('Rudolph Valentino', 'people.person.profession', 'Film Producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jagged Edge'), ('The Island Def Jam Music Group', 'common.topic.notable_for', 'g.125b9qymk'), ('Daniel Bedingfield', 'people.person.gender', 'Male'), ('Ginuwine', 'people.person.profession', 'Dancer'), ('Michael Jackson', 'broadcast.artist.content', 'radioIO Classic RNB'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.featured_artists', 'Big Sean'), ('Confident', 'music.album.primary_release', 'Confident'), ('Jordin Sparks', 'people.person.profession', 'Artist'), ('Brandy Norwood', 'people.person.profession', 'Dancer'), ('Justin Timberlake', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Kid Cudi', 'people.person.profession', 'Singer-songwriter'), ('Mariah Carey', 'music.artist.genre', 'Pop music'), ('Heartbreaker', 'music.album.artist', 'Justin Bieber'), ('Place of birth', 'rdf-schema#range', 'Location'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Cory Gunz', 'music.artist.label', 'The Island Def Jam Music Group'), ('School Boy Records', 'music.record_label.artist', 'Rixton'), ('Zendaya: Behind the Scenes', 'film.film.personal_appearances', 'm.0w3gbtv'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Shaffer Smith'), ('HitzRadio.com', 'broadcast.content.artist', '112'), ('justinbieber', 'common.topic.notable_for', 'g.1q3sdnv9d'), ('Love Me', 'music.album.release_type', 'Single'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.distributors', 'm.0hvlt03'), ('City/Town/Village', 'freebase.type_profile.strict_included_types', 'Location'), ('Kanye West', 'broadcast.artist.content', 'Smoothbeats'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101ftp1', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Hip hop music', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('My Worlds', 'music.album.releases', 'My Worlds'), ('2011 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw444'), ('m.0101fv8j', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Jason Mraz', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('m.0gbm3d3', 'film.personal_film_appearance.person', 'Ludacris'), ('1Club.FM: Mix 106', 'common.topic.notable_types', 'Broadcast Content'), ('R. Kelly', 'people.person.profession', 'Music executive'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Nicki Minaj', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Right Here (featuring Drake)', 'music.recording.artist', 'Drake'), ('Live My Life', 'music.album.featured_artists', 'Redfoo'), ('m.0j1t2xs', 'base.firsts.first_achievement.category', 'Canadian'), ('m.0d_hbgr', 'common.webpage.resource', 'Justin Bieber Lyrics'), ('Michael Jackson', 'broadcast.artist.content', '1Club.FM: V101'), ('Never Let You Go', 'common.topic.notable_for', 'g.12h312yf9'), ('World music', 'music.genre.subgenre', 'Reggae'), ('Diplo', 'common.topic.notable_types', 'Musical Artist'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Sonique'), (\"Justin Bieber's Believe\", 'film.film.music', 'Nathan Lanier'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'William Orbit'), ('Pras', 'music.artist.genre', 'Reggae'), ('m.0z85n_1', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Timbaland', 'music.artist.genre', 'Contemporary R&B'), ('m.0ng_j6d', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Singer', 'people.profession.specializations', 'Classical Vocalist'), ('Snoop Dogg', 'people.person.profession', 'Singer-songwriter'), ('HitzRadio.com', 'broadcast.content.artist', 'Three Days Grace'), ('m.0gh0fdt', 'tv.tv_guest_role.episodes_appeared_in', 'Series 1, Show 14'), ('Nicki Minaj', 'music.artist.genre', 'Dance-pop'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'BWO'), ('m.0yr9tjb', 'award.award_nomination.ceremony', '2010 Young Hollywood Awards'), ('Mason Levy', 'people.person.gender', 'Male'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftmz'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Nina Sky'), ('Mason Levy', 'people.person.nationality', 'United States of America'), ('Wait for a Minute', 'music.composition.composer', 'Maejor'), ('Rodney Jerkins', 'music.composer.compositions', 'As Long as You Love Me'), ('Never Let You Go', 'common.topic.notable_types', 'Musical Album'), ('Live My Life', 'common.topic.notable_types', 'Composition'), ('Justin Timberlake', 'broadcast.artist.content', 'HitzRadio.com'), ('Whitney Houston', 'people.person.nationality', 'United States of America'), ('Dance music', 'broadcast.genre.content', 'JellyRadio.com'), ('m.0101fsz2', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Ellen DeGeneres'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Rock music'), ('Chingy', 'music.artist.genre', 'Dance-pop'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Black Eyed Peas', 'music.artist.genre', 'Contemporary R&B'), ('m.0101ftt8', 'film.film_crew_gig.crewmember', 'Kiyomi Hara'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Gavin DeGraw'), ('m.0v30srx', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Iggy Azalea', 'music.artist.label', 'The Island Def Jam Music Group'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('Composition', 'freebase.type_profile.strict_included_types', 'Topic'), ('Max Martin', 'music.artist.label', 'Island Records'), ('m.09y886k', 'common.webpage.topic', 'Teen idol'), ('Concert tour', 'freebase.type_hints.included_types', 'Topic'), ('Demi Lovato', 'people.person.profession', 'Dancer'), ('Never Say Never: The Remixes', 'common.topic.image', '518hQytx0BL.jpg'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('m.0101fvl8', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Justin Timberlake', 'broadcast.artist.content', '181-beat'), ('#thatPOWER', 'common.topic.notable_types', 'Canonical Version'), ('Jordan Pruitt', 'people.person.profession', 'Singer-songwriter'), ('m.0njdns_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Lil Wayne', 'people.person.profession', 'Actor'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Selena Gomez', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsc'), ('m.09wjs_l', 'common.webpage.topic', 'Record producer'), ('Bryan-Michael Cox', 'people.person.nationality', 'United States of America'), ('Sean Kingston', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3cp'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Gwen Stefani'), ('Lady Gaga', 'music.artist.genre', 'Electronic dance music'), ('Juvenile', 'people.person.profession', 'Musician'), ('Eenie Meenie', 'common.topic.notable_for', 'g.12h2y4ysq'), ('Chicago', 'location.hud_county_place.place', 'Chicago'), ('m.0njw59_', 'award.award_honor.award', 'MTV Europe Music Award for Best Pop'), ('Shaggy', 'music.artist.genre', 'Reggae'), ('Confident', 'music.album.featured_artists', 'Chance the Rapper'), ('m.0101ft35', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Demi Lovato', 'music.artist.genre', 'Synthpop'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'common.topic.notable_types', 'Broadcast Content'), ('Red Hot Chili Peppers', 'music.artist.genre', 'Rock music'), ('1.FM Top 40', 'broadcast.content.artist', 'Anastacia'), ('Right Here', 'music.recording.song', 'Right Here'), ('Gavin DeGraw', 'people.person.profession', 'Singer-songwriter'), ('m.0101ft0d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Madonna', 'broadcast.artist.content', '.977 The Hits Channel'), ('Donna Summer', 'music.artist.genre', 'Contemporary R&B'), ('2010 Young Hollywood Awards', 'award.award_ceremony.nominees', 'm.0yr9tjb'), ('Akon', 'people.person.languages', 'English Language'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ian Oliver'), ('Blu Cantrell', 'people.person.profession', 'Singer'), ('Recovery', 'common.topic.notable_for', 'g.1yg4dl30h'), ('m.0nfnx_3', 'film.personal_film_appearance.film', 'Bad 25'), ('Teen idol', 'base.icons.icon_genre.icons', 'Neil Sedaka'), ('Book', 'freebase.type_profile.strict_included_types', 'Topic'), ('Rita Ora', 'music.artist.genre', 'Contemporary R&B'), ('181-beat', 'broadcast.content.artist', 'Rihanna'), ('Jordan Pruitt', 'people.person.profession', 'Singer'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Mason Levy', 'music.artist.genre', 'Pop music'), ('m.0njw257', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Sheryl Crow', 'people.person.profession', 'Musician'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nasri', 'people.person.gender', 'Male'), (\"Destiny's Child\", 'broadcast.artist.content', 'HitzRadio.com'), ('Juno Awards of 2013', 'award.award_ceremony.awards_presented', 'm.0t4s_bn'), ('Mariah Carey', 'broadcast.artist.content', '1Club.FM: V101'), ('Rihanna', 'broadcast.artist.content', 'radioIO Todays RNB'), ('m.0101fszb', 'film.personal_film_appearance.person', 'Usher'), ('Jordan Francis', 'people.person.nationality', 'Canada'), ('Lolly', 'music.single.versions', 'Lolly'), ('Fabolous', 'broadcast.artist.content', '181-beat'), ('Mannie Fresh', 'people.person.profession', 'Musician'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Contemporary R&B'), ('Hot Wired Radio', 'broadcast.content.artist', 'Zac Brown Band'), ('Person', 'type.type.properties', 'Children'), ('Justin Bieber', 'music.composer.compositions', 'Die in Your Arms'), ('Britney Spears', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Madonna', 'people.person.profession', 'Dancer'), ('Juvenile', 'people.person.nationality', 'United States of America'), ('HitzRadio.com', 'broadcast.content.artist', 'Ginuwine'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('1Club.FM: V101', 'broadcast.content.artist', 'R. Kelly'), ('Britney Spears', 'music.artist.genre', 'Contemporary R&B'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Rock music'), ('Dr. Dre', 'people.person.profession', 'Musician'), ('Khalil', 'freebase.valuenotation.has_value', 'Parents'), ('Bigger', 'music.composition.lyricist', 'Frank Ocean'), ('Teen idol', 'base.icons.icon_genre.icons', 'Leonardo DiCaprio'), ('Will Smith', 'people.person.profession', 'Film Producer'), ('Juelz Santana', 'people.person.profession', 'Actor'), ('Mann', 'music.artist.genre', 'Hip hop music'), ('m.0gwhmhm', 'award.award_honor.ceremony', 'Juno Awards of 2011'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92g'), ('Baby', 'music.composition.recordings', 'Baby'), ('Justin Bieber', 'music.composer.compositions', 'Never Say Never'), ('Yves Bole', 'people.person.nationality', 'Italy'), ('1.FM Top 40', 'broadcast.content.artist', 'LeAnn Rimes'), ('181-beat', 'broadcast.content.artist', 'Baby Bash'), ('School Boy Records', 'music.record_label.artist', 'Asher Roth'), ('Ludacris', 'music.featured_artist.albums', 'All Around the World'), ('JoJo', 'music.artist.genre', 'Teen pop'), ('FLOW 103', 'broadcast.content.artist', 'The Pussycat Dolls'), ('m.0101fsyr', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Gavin DeGraw', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0_srv2b'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Snoop Dogg', 'people.person.profession', 'Actor'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Ashlee Simpson', 'people.person.profession', 'Actor'), ('m.0gbmnsy', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Lady Antebellum', 'broadcast.artist.content', 'radioIO Todays POP'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0_grqb8'), ('m.0pcr2qn', 'tv.tv_guest_role.character', \"Pizzi's Best Friend\"), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pray', 'music.album.primary_release', 'Pray'), ('HitzRadio.com', 'broadcast.content.artist', 'M.I.A.'), ('m.0ng_vkd', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Chris Brown', 'people.person.profession', 'Record producer'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Benny Blanco', 'music.composer.compositions', 'Eenie Meenie'), ('All That Matters', 'music.album.release_type', 'Single'), ('Tyga', 'award.award_nominee.award_nominations', 'm.0_xlw5f'), ('Snoop Dogg', 'broadcast.artist.content', 'Smoothbeats'), ('1.FM Top 40', 'broadcast.content.artist', 'Beverley Knight'), ('London', 'location.location.containedby', 'Area codes 519 and 226'), ('Cris Cab', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber', 'music.composer.compositions', 'Bigger'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('Thought of You', 'music.single.versions', 'Thought Of You'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'The All-American Rejects'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsc'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sean Paul'), ('Sean Combs', 'music.artist.genre', 'Synthpop'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot instrumental)'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Cascada'), ('1Club.FM: V101', 'broadcast.content.artist', 'Michael Jackson'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Rihanna', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('#thatPOWER', 'common.topic.notable_for', 'g.11b5lrbn8m'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvjm'), (\"Nickelodeon Mexico Kids' Choice Award for Favorite Song\", 'award.award_category.winners', 'm.0yrjkl1'), ('Beauty and a Beat', 'music.album.compositions', 'Beauty And A Beat'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordan Pruitt', 'music.artist.genre', 'Teen pop'), (\"Bill O'Dowd\", 'people.person.gender', 'Male'), ('Hot Wired Radio', 'common.topic.article', 'm.03hrz8s'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Sean Kingston', 'people.person.gender', 'Male'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('My World 2.0', 'music.album.genre', 'Rhythm and blues'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Alicia Keys', 'people.person.gender', 'Female'), ('Akon', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Hold Tight', 'music.recording.artist', 'Justin Bieber'), ('PYD', 'music.album.artist', 'Justin Bieber'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0n1ykxp', 'award.award_honor.award_winner', 'Ludacris'), ('Kuk Harrell', 'music.composer.compositions', 'Never Say Never'), ('Canada', 'location.country.administrative_divisions', 'Ontario'), ('radioIO Todays POP', 'broadcast.content.location', 'Tampa'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjkl1'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ice Cube'), ('Favorite Girl', 'music.album.primary_release', 'Favorite Girl'), ('Chef Tone', 'music.artist.genre', 'Contemporary R&B'), ('Beauty And A Beat', 'common.topic.notable_types', 'Composition'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.featured_artists', 'Big Sean'), ('m.0tjyljn', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('RedOne', 'music.artist.genre', 'Hip hop music'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0yrhrwc', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0t_l7mp'), ('All Around The World', 'music.composition.place_of_first_performance', 'United States of America'), ('All Around The World', 'music.composition.recordings', 'All Around The World (featuring Ludacris)'), ('Right Here (featuring Drake)', 'music.recording.song', 'Right Here'), ('Christina Aguilera', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'freebase.valuenotation.has_no_value', 'Weblink'), ('m.0gbm3cx', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Beautiful', 'music.recording.featured_artists', 'Justin Bieber'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs16'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.artist', 'Justin Bieber'), ('Cory Gunz', 'people.person.profession', 'Singer'), ('Katy Perry', 'music.artist.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Ricky Nelson'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Classic hits'), ('Justin Bieber', 'celebrities.celebrity.legal_entanglements', 'm.0_cz5l3'), ('Kelly Clarkson', 'people.person.gender', 'Female'), ('PYD', 'music.album.releases', 'PYD'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Confident', 'music.composition.composer', 'Soundz'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Chris Brown'), ('2012 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0jvgmxc'), ('Akon', 'people.person.profession', 'Singer'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Children'), ('Change Me', 'common.topic.notable_for', 'g.1yn5bkttb'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Khalil'), ('Little Bird', 'music.recording.tracks', 'Little Bird'), ('Justin Bieber', 'music.composer.compositions', 'All Around The World'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ramon Zerano'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Pop music'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaffer Smith', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'common.topic.image', 'beirutnights.png'), ('Mariah Carey', 'people.person.profession', 'Model'), ('C1', 'people.person.gender', 'Male'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Miley Cyrus', 'people.person.profession', 'Actor'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j219nq'), ('PowerHitz', 'broadcast.content.genre', 'Rap music'), ('Hikaru Utada', 'people.person.profession', 'Record producer'), ('#thatPOWER', 'music.single.versions', '#thatPower'), ('LeAnn Rimes', 'people.person.profession', 'Singer'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Tonic'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Fabolous'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Snoop Dogg', 'music.artist.genre', 'Hip hop music'), ('Katy Perry: Part of Me', 'film.film.genre', 'Documentary film'), ('Will i Am', 'music.artist.genre', 'Dance-pop'), ('Beyoncé Knowles', 'music.artist.genre', 'Pop music'), ('Eminem', 'people.person.gender', 'Male'), ('Place of birth', 'rdf-schema#domain', 'Person'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z1jn32'), ('m.0_vmmj6', 'award.award_nomination.award_nominee', 'Nicki Minaj'), ('1.FM Top 40', 'broadcast.content.artist', '112'), ('Reed Smoot', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Live My Life', 'music.album.album_content_type', 'Studio album'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Tupac Shakur', 'broadcast.artist.content', 'FLOW 103'), (\"Dick Clark's Primetime New Year's Rockin' Eve 2013\", 'film.film.personal_appearances', 'm.0pcnqnb'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Italy', 'location.country.languages_spoken', 'Italian Language'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten radio)'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c920'), ('m.0njhx1b', 'award.award_honor.award', 'Billboard Music Award for Top Streaming Artist'), ('1Club.FM: Power', 'broadcast.content.artist', 'M.I.A.'), ('Believe Acoustic', 'music.album.artist', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fall Out Boy'), ('Shaffer Smith', 'broadcast.artist.content', 'JellyRadio.com'), ('Bad Day', 'music.composition.composer', 'James Giannos'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Never Say Never', 'common.topic.notable_types', 'Composition'), ('m.0v90p2b', 'award.award_honor.award_winner', 'Justin Bieber'), ('Record producer', 'freebase.equivalent_topic.equivalent_type', 'Record Producer'), ('Mason Levy', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0p85jpp'), ('Confident', 'music.composition.recordings', 'Confident'), ('Contemporary R&B', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), ('Lolly', 'common.topic.notable_for', 'g.1yfp2v185'), ('m.0z84kwx', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.featured_artist.recordings', 'Where Are Ü Now'), ('Teen idol', 'base.icons.icon_genre.icons', 'Ashley Tisdale'), ('m.0_vmmj6', 'award.award_nomination.ceremony', '2013 Radio Disney Music Awards'), ('m.0z2dr9y', 'award.award_nomination.nominated_for', 'justinbieber'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('m.0p85jpp', 'film.personal_film_appearance.film', 'Katy Perry: Part of Me'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Enrique Iglesias', 'broadcast.artist.content', '1.FM Top 40'), ('m.0gbm3fr', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Chef Tone', 'music.composer.compositions', 'Heartbreaker'), ('Bad Day', 'music.composition.recordings', 'Bad Day'), ('Bad Day', 'music.composition.composer', 'Chris Jasper'), ('Ray J', 'music.artist.genre', 'Hip hop music'), ('American Music Award for Favorite Pop/Rock Album', 'award.award_category.winners', 'm.0njgyk4'), ('U Smile', 'music.album.artist', 'Justin Bieber'), ('Judy Garland', 'people.person.profession', 'Singer'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('m.09y8cm9', 'common.webpage.topic', 'Teen idol'), ('Kanye West', 'broadcast.artist.content', 'Hot 97.7'), ('R. Kelly', 'music.featured_artist.albums', 'PYD'), ('Usher', 'broadcast.artist.content', 'WildFMRadio.com'), ('Never Say Never (acoustic)', 'music.recording.canonical_version', 'Never Say Never'), ('Snoop Dogg', 'broadcast.artist.content', 'JellyRadio.com'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Ethnicity', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Ginuwine', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Big Sean', 'people.person.profession', 'Singer'), ('Justin Bieber', 'people.person.ethnicity', 'Canadian'), ('Miley Cyrus', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Hot 108 Jamz', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0kyjls4'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mario Winans'), ('2012 Teen Choice Awards', 'time.event.people_involved', 'Demi Lovato'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pattie Mallette', 'common.topic.article', 'm.0n40czg'), ('Brandy Norwood', 'music.artist.genre', 'Contemporary R&B'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Shaffer Smith'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mark Morrison'), ('HitzRadio.com', 'broadcast.content.artist', 'Gavin DeGraw'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Place of birth'), ('Pattie Mallette', 'people.person.languages', 'g.11byb5qj3x'), ('m.0njhvsq', 'freebase.valuenotation.has_no_value', 'Winning work'), ('This is Justin Bieber', 'film.film.genre', 'Documentary film'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Ronald Isley'), ('Madonna', 'music.artist.genre', 'Electronic music'), ('Nelly Furtado', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Timberlake', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Wait For a Minute', 'music.album.release_type', 'Single'), ('L.A. Reid', 'common.topic.notable_types', 'Record Producer'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO RNB Mix'), ('As Long as You Love Me (album version)', 'music.recording.featured_artists', 'Big Sean'), ('HitzRadio.com', 'broadcast.content.artist', 'Ciara'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty And A Beat', 'common.topic.article', 'm.0k2b595'), ('Fergie', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctps1'), ('Sia Furler', 'music.artist.genre', 'Pop music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Sir Mix-a-Lot'), ('Teyana', 'music.artist.genre', 'Pop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Jimmy Eat World'), ('Under the Mistletoe', 'music.album.genre', 'Holiday'), ('Benny Blanco', 'music.artist.genre', 'Contemporary R&B'), ('Teen idol', 'common.topic.webpage', 'm.09y886k'), ('Johntá Austin', 'music.artist.genre', 'Hip hop music'), ('m.0y803nt', 'award.award_honor.ceremony', '2011 Brit Awards'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty And A Beat', 'music.composition.recordings', 'Beautiful and the Beat'), ('m.0vp7m_x', 'music.recording_contribution.contributor', 'Raekwon'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lady Antebellum'), ('m.0z340zt', 'award.award_honor.award_winner', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.rating', 'PG (USA)'), ('Baby', 'award.award_winning_work.awards_won', 'm.0yrjkl1'), ('David Cassidy', 'base.icons.icon.icon_genre', 'Teen idol'), ('School Boy Records', 'music.record_label.artist', 'Psy'), ('181-beat', 'common.topic.notable_types', 'Broadcast Content'), ('Yves Bole', 'freebase.valuenotation.is_reviewed', 'Notable types'), ('Kylie Minogue', 'music.artist.genre', 'Electronic music'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'AEG Live'), ('m.043082k', 'common.webpage.category', 'Topic Webpage'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Gwen Stefani', 'people.person.profession', 'Record producer'), ('August Rigo', 'music.artist.label', 'The Island Def Jam Music Group'), ('Leonardo DiCaprio', 'people.person.nationality', 'United States of America'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0gbmnt3', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Pop music'), ('Contemporary R&B', 'broadcast.genre.content', 'Dif.'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('Jessica Simpson', 'music.artist.genre', 'Contemporary R&B'), ('Miami', 'base.biblioness.bibs_location.country', 'United States of America'), ('Ricky Nelson', 'people.person.profession', 'Actor'), ('Recovery', 'common.topic.notable_types', 'Musical Album'), ('m.0z87597', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0vp800w', 'music.recording_contribution.contributor', 'Ludacris'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('All That Matters', 'common.topic.notable_types', 'Composition'), ('All Around The World (featuring Ludacris)', 'music.recording.releases', 'Believe'), ('m.012nv5h3', 'people.place_lived.person', 'Yves Bole'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Year'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Gender'), ('London', 'base.wikipedia_infobox.settlement.area_code', 'Area codes 519 and 226'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Fabolous'), ('School Boy Records', 'music.record_label.artist', 'Martin Garrix'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'September'), ('Terius Nash', 'people.person.profession', 'Singer-songwriter'), ('Katy Perry', 'music.artist.genre', 'Dance music'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.personal_appearances', 'm.0j_tkcq'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k21'), ('Michael Jackson', 'people.person.nationality', 'United States of America'), ('Lolly', 'music.recording.song', 'Lolly'), ('Justin Bieber', 'music.artist.album', 'All Around the World'), ('m.0101fv9w', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Spouse (or domestic partner)', 'type.property.master_property', 'Spouse'), ('m.0101ft5f', 'film.personal_film_appearance.person', 'Kuk Harrell'), ('Tupac Shakur', 'people.person.languages', 'English Language'), ('HitzRadio.com', 'broadcast.content.artist', 'Fabolous'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Height'), ('Timbaland', 'people.person.profession', 'Actor'), ('1Club.FM: Power', 'broadcast.content.artist', 'Trick Daddy'), ('JellyRadio.com', 'broadcast.content.genre', 'Pop music'), ('Ciara', 'people.person.profession', 'Actor'), ('Jaden Smith', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3d_'), ('m.0yqflhy', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('Nathan Lanier', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0j_tkcq'), ('Kanye West', 'music.artist.genre', 'Contemporary R&B'), ('Somebody to Love (J Stax remix)', 'common.topic.notable_types', 'Musical Recording'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Singer'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0yqfny6', 'award.award_honor.award_winner', 'Justin Bieber'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Janet Jackson', 'music.artist.genre', 'Pop music'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0y_g556'), ('Lil Jon', 'broadcast.artist.content', '1Club.FM: Power'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Paula DeAnda'), ('Teen idol', 'common.topic.webpage', 'm.09y8bsk'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Height'), ('Lil Wayne', 'people.person.nationality', 'United States of America'), ('Jordin Sparks', 'people.person.profession', 'Musician'), ('Jane Lipsitz', 'freebase.valuenotation.has_value', 'Parents'), ('Never Say Never: The Remixes', 'music.album.artist', 'Justin Bieber'), ('Live My Life', 'music.single.versions', 'Little Bird'), ('Akon', 'broadcast.artist.content', '.977 The Hits Channel'), ('PYD', 'common.topic.notable_types', 'Musical Recording'), ('Hot Wired Radio', 'broadcast.content.artist', 'The Fray'), ('P!nk', 'music.artist.genre', 'Dance music'), ('Justin Timberlake', 'music.artist.genre', 'Dance-pop'), ('Alicia Keys', 'people.person.profession', 'Singer-songwriter'), ('#Thatpower', 'music.recording.song', '#thatPower'), ('m.0zg2qjr', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('m.0njhxzc', 'award.award_honor.honored_for', 'My World 2.0'), ('m.0njvtth', 'award.award_honor.award', 'Juno Award for Pop Album of the Year'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('HitzRadio.com', 'broadcast.content.artist', 'Flyleaf'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Fergie'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Somebody to Love (J Stax remix)', 'music.recording.song', 'Somebody to Love'), ('Never Say Never: The Remixes', 'music.album.genre', 'Dance-pop'), ('Jason Mraz', 'music.artist.genre', 'Reggae'), ('Mariah Carey', 'people.person.profession', 'Film Producer'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0ywtfj6'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Biographical Documentaries'), ('All Around The World', 'common.topic.notable_types', 'Composition'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrk18w'), ('Tricky Stewart', 'freebase.valuenotation.has_value', 'Parents'), ('m.0y84ynj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Beauty and a Beast', 'music.recording.featured_artists', 'Nicki Minaj'), ('WildFMRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Ja Rule', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0njwb81', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105zs8k'), ('#thatPOWER', 'music.single.versions', '#Thatpower'), ('Dr. Dre', 'broadcast.artist.content', 'Smoothbeats'), ('m.0b48fj5', 'common.webpage.topic', 'Record producer'), ('Big Sean', 'music.composer.compositions', 'As Long as You Love Me'), ('Baby', 'music.composition.composer', 'Terius Nash'), ('Miley Cyrus', 'broadcast.artist.content', 'Hot Wired Radio'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Christina Aguilera', 'people.person.profession', 'Singer-songwriter'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Michael Jackson', 'influence.influence_node.influenced', 'Britney Spears'), ('Beyoncé Knowles', 'broadcast.artist.content', '.977 The Hits Channel'), ('Profession', 'type.property.expected_type', 'Profession'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkw8v'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.song', 'Beauty And A Beat'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Baby Bash'), ('Live My Life', 'music.single.versions', 'Live My Life (Party Rock remix)'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Children'), ('Mariah Carey', 'people.person.profession', 'Record producer'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien dubstep mix)'), ('Pop music', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('All Around the World', 'music.album.release_type', 'Single'), ('Dancer', 'music.special_music_video_performance_type.special_music_video_performances', 'm.0z3vx9q'), ('Kelly Clarkson', 'people.person.nationality', 'United States of America'), ('Nelly', 'people.person.profession', 'Singer-songwriter'), ('NME Award for Worst Album', 'award.award_category.nominees', 'm.0z8t2dy'), ('m.0y_g42w', 'award.award_nomination.nominated_for', 'justinbieber'), ('Max Martin', 'music.artist.genre', 'Dance music'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Release type'), ('WildFMRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Hot Wired Radio', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('2010 Young Hollywood Awards', 'award.award_ceremony.awards_presented', 'm.0yr9c1k'), ('Whitney Houston', 'music.artist.genre', 'Dance-pop'), ('Anastacia', 'music.artist.label', 'The Island Def Jam Music Group'), ('Believe Acoustic', 'common.topic.notable_for', 'g.126tqpnyl'), ('Justin Bieber', 'common.topic.webpage', 'm.07053l5'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt8'), ('Gavin DeGraw', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Hip hop music', 'broadcast.genre.content', 'Hot 97.7'), ('m.0njhtjj', 'award.award_honor.award', 'Billboard Music Award for Top New Artist'), ('m.0y_g556', 'award.award_honor.ceremony', '4th Annual Shorty Awards'), ('1.FM Top 40', 'broadcast.content.artist', 'Van Halen'), ('Brandy Norwood', 'people.person.profession', 'Record producer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ferry Corsten'), ('2010 MuchMusic Video Awards', 'award.award_ceremony.awards_presented', 'm.0njwb81'), ('Chris Brown', 'people.person.gender', 'Male'), ('m.0ng_vkd', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Blu Cantrell', 'broadcast.artist.content', 'DeeGay'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'Boys Like Girls'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Mary J. Blige', 'broadcast.artist.content', '1Club.FM: V101'), ('m.0njhtjj', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kanye West', 'people.person.gender', 'Male'), ('As Long As You Love Me (Ferry Corsten remix)', 'music.recording.featured_artists', 'Big Sean'), ('Baby', 'music.composition.recordings', 'Baby'), ('Amerie', 'music.artist.genre', 'K-pop'), ('This is Justin Bieber', 'film.film.personal_appearances', 'm.0ng_j6d'), ('Baby', 'music.composition.language', 'English Language'), ('Singer', 'people.profession.specializations', 'Singer-songwriter'), ('Die in Your Arms', 'music.composition.composer', 'Thomas Lumpkins'), ('Urban contemporary', 'broadcast.genre.content', 'Smoothbeats'), ('Jason Mraz', 'music.artist.genre', 'Acoustic music'), ('Live My Life (Party Rock remix)', 'common.topic.notable_types', 'Musical Recording'), ('Jay-Z', 'broadcast.artist.content', 'Smoothbeats'), ('Keyshia Cole', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: V101', 'broadcast.content.artist', 'Alicia Keys'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('Duffy', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Alicia Keys', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0102z0vx'), ('Katy Perry', 'music.artist.label', 'Island Records'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0t_l7mp'), ('My World', 'common.topic.article', 'm.0805zjd'), ('Bryan Adams', 'music.artist.genre', 'Pop music'), ('Scooter Braun', 'music.artist.label', 'School Boy Records'), ('Boyfriend (Dada Life remix)', 'music.recording.canonical_version', 'Boyfriend'), ('Terius Nash', 'music.artist.genre', 'Contemporary R&B'), ('Fergie', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Diplo', 'freebase.valuenotation.has_value', 'Parents'), ('Sean Combs', 'broadcast.artist.content', '181-beat'), ('Runaway Love (remix)', 'music.recording.canonical_version', 'Runaway Love'), ('Lady Gaga', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('RedOne', 'music.artist.genre', 'Pop music'), ('Katy Perry', 'music.artist.genre', 'Dance-pop'), ('Juelz Santana', 'people.person.profession', 'Musician'), ('Rihanna', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Gwen Stefani', 'music.artist.genre', 'Electronic music'), ('Dance music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('Avery', 'music.artist.genre', 'Synthpop'), ('Will Smith', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Recovery', 'music.composition.composer', 'Craig David'), ('Geri Halliwell', 'music.artist.genre', 'Pop music'), ('Johntá Austin', 'people.person.profession', 'Record producer'), ('KooL CrAzE', 'music.artist.genre', 'Hip hop music'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Parents'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Simple Plan'), ('Person', 'freebase.type_hints.included_types', 'Topic'), (\"Pizzi's Best Friend\", 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr2qn'), ('Justin Bieber', 'music.artist.contribution', 'm.0vp7mrq'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Next to You', 'common.topic.notable_for', 'g.12lqg_0ly'), ('Ashley Tisdale', 'music.artist.genre', 'Dance-pop'), ('m.0ng_k21', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('Camagüey', 'location.location.time_zones', 'Eastern Time Zone'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Music'), ('m.0jztshx', 'film.performance.actor', 'Justin Bieber'), ('Contemporary R&B', 'broadcast.genre.content', 'Hot 97.7'), ('Chloe Bridges', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber: Never Say Never', 'common.topic.image', '6759bc3bDlbiBmi_1_l.jpg'), ('Ice Cube', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Baby', 'music.recording.canonical_version', 'Baby'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjmq'), ('World music', 'broadcast.genre.content', 'DeeGay'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0z340zt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Hot Wired Radio', 'broadcast.content.genre', 'Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gsj'), ('Justin Timberlake', 'people.person.profession', 'Film Producer'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0cq990t', 'common.webpage.resource', 'Justin Bieber Romania'), ('Tupac Shakur', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Usher'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Justin Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Hot Wired Radio', 'broadcast.content.genre', 'Hip hop music'), ('Pras', 'people.person.nationality', 'United States of America'), ('Believe', 'music.album.primary_release', 'Believe'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.composition.composer', 'Christina Milian'), ('Chris Brown', 'broadcast.artist.content', 'PowerHitz'), ('Nick Jonas', 'music.artist.genre', 'Contemporary R&B'), ('Sunshine Radio', 'broadcast.content.genre', 'International'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Children'), ('Chris Brown', 'music.artist.genre', 'Urban contemporary'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Miley Cyrus'), ('Beauty and a Beat', 'music.recording.artist', 'Nicki Minaj'), ('All Around the World', 'music.album.featured_artists', 'Ludacris'), ('Selena Gomez', 'music.artist.genre', 'Rock music'), ('Kylie Minogue', 'people.person.profession', 'Record producer'), ('m.0gbm3d3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Ellen DeGeneres', 'people.person.gender', 'Female'), ('Redfoo', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h523f'), ('All That Matters', 'music.composition.recordings', 'All That Matters (video)'), ('Never Say Never', 'music.album.featured_artists', 'Jaden Smith'), ('Thought Of You', 'common.topic.notable_types', 'Musical Recording'), ('50 Cent', 'broadcast.artist.content', 'WildFMRadio.com'), ('Shaggy', 'people.person.profession', 'Actor'), ('m.0z898w6', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('m.0gbm35z', 'film.film_cut.film_release_region', 'United States of America'), ('Vanessa Hudgens', 'music.artist.genre', 'Teen pop'), ('Eminem', 'people.person.profession', 'Actor'), ('Johntá Austin', 'people.person.gender', 'Male'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Bad Day', 'music.album.release_type', 'Single'), ('m.0z3tqqt', 'award.award_nomination.award', 'Shorty Award for Music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_j6d'), ('Teen Choice Award for Choice Music: Album Pop', 'award.award_category.nominees', 'm.0z8qqh5'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Love Never Felt So Good', 'common.topic.notable_types', 'Musical Album'), ('Yves Bole', 'influence.influence_node.influenced', 'iJustine'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0jvgmxc', 'award.award_honor.ceremony', '2012 Billboard Music Awards'), ('R. Kelly', 'music.artist.genre', 'Contemporary R&B'), ('Foreign Remix', 'music.recording.tracks', 'Foreign Remix'), ('30 Days in May', 'common.topic.notable_types', 'Film'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('BeirutNights.com Radio', 'broadcast.content.broadcast', 'BeirutNights.com Radio - 128kbps Stream'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Italy', 'location.country.official_language', 'Italian Language'), ('Clay Aiken', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0njdq6k', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0z8qx3w', 'award.award_honor.award', 'Teen Choice Award for Choice Music: Album Pop'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Dance-pop', 'common.topic.notable_types', 'Musical genre'), ('Under the Mistletoe', 'music.album.releases', 'Under the Mistletoe'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'September'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Musical Artist', 'type.type.expected_by', 'Artists'), ('Chris Brown', 'people.person.profession', 'Dancer'), ('m.0yrjkl1', 'award.award_honor.award', \"Nickelodeon Mexico Kids' Choice Award for Favorite Song\"), ('Beauty And A Beat', 'music.composition.composer', 'Zedd'), ('m.0njgyk4', 'award.award_honor.award_winner', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('m.0101ftl5', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Big Sean'), ('R. Kelly', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Canada', 'location.country.languages_spoken', 'English Language'), ('1.FM Top 40', 'broadcast.content.artist', 'Max Graham vs. Yes'), ('The Roots', 'broadcast.artist.content', '1Club.FM: V101'), ('Janet Jackson', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Adrienne Bailon', 'people.person.profession', 'Actor'), ('Teen idol', 'base.icons.icon_genre.icons', 'David Cassidy'), ('Foreign Remix', 'music.recording.featured_artists', 'Justin Bieber'), ('Musical Artist', 'freebase.type_profile.equivalent_topic', 'Musician'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0z1jn32', 'award.award_nomination.nominated_for', 'justinbieber'), ('School Gyrls', 'film.film.country', 'United States of America'), ('Ciara', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.artist', 'Justin Bieber'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO Todays POP'), ('Singer', 'common.topic.subject_of', 'Léo Ferré'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cg'), ('Sean Kingston', 'music.artist.album', 'Eenie Meenie'), ('Pray', 'music.composition.recordings', 'Pray'), ('m.0jvgmxc', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jazmyn Bieber', 'people.person.nationality', 'Canada'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('U Smile', 'common.topic.notable_for', 'g.12h2_3_z_'), ('Baby', 'music.recording.song', 'Baby'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('m.0t4syfh', 'award.award_nomination.ceremony', 'Juno Awards of 2012'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Willa Ford'), ('m.0j8z6tl', 'award.award_nomination.ceremony', '2012 Billboard Music Awards'), ('Teen idol', 'base.icons.icon_genre.icons', 'Josue Diaz'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Rudolph Valentino', 'people.person.profession', 'Dancer'), ('m.07053l5', 'common.webpage.category', 'Topic Webpage'), ('Where Are Ü Now', 'music.recording.artist', 'Jack Ü'), ('Khalil', 'music.artist.genre', 'Hip hop music'), ('Justin Bieber', 'music.artist.album', 'Under the Mistletoe'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.song', 'As Long as You Love Me'), ('Jeremy Bieber', 'people.person.profession', 'Musician'), ('m.0njhyh_', 'award.award_honor.honored_for', 'Baby'), ('PowerHitz', 'broadcast.content.artist', 'Usher'), ('Lady Gaga', 'music.artist.genre', 'Dance music'), ('radioIO Todays POP', 'broadcast.content.genre', 'Pop music'), ('m.0yrlqp1', 'education.education.student', 'Justin Bieber'), ('Rita Ora', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Christina Aguilera'), ('Alicia Keys', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.music', 'Deborah Lurie'), ('Santana', 'music.artist.genre', 'Rock music'), ('NME Award for Worst Album', 'award.award_category.winners', 'm.0z8s_wn'), ('Colbie Caillat', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'theater.theater_company.plays_produced', 'Def Poetry'), ('Jay-Z', 'people.person.profession', 'Actor'), ('Baby', 'music.album.releases', 'Baby'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Britney Spears'), ('William Orbit', 'common.topic.notable_types', 'Musical Artist'), ('Drake', 'people.person.profession', 'Singer'), ('Emphatic Radio.com!', 'common.topic.notable_for', 'g.1255_9wpq'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Cassie Ventura'), ('Radio Disney Music Award for Best Male Artist', 'award.award_category.winners', 'm.0y4tdml'), ('Justin Bieber', 'music.featured_artist.albums', '#thatPOWER'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'No Doubt'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Keyshia Cole', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'people.person.profession', 'Singer'), ('Kelis', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('iJustine', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('Lady Antebellum', 'music.artist.genre', 'Pop music'), ('Sir Nolan', 'people.person.nationality', 'United States of America'), ('Max Martin', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.single.versions', 'Boyfriend'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.has_value', 'Parents'), ('m.0z8s_wn', 'award.award_honor.award', 'NME Award for Worst Album'), ('Michael Jackson', 'influence.influence_node.influenced', 'Justin Timberlake'), ('School Gyrls', 'film.film.produced_by', 'L.A. Reid'), ('Nelly', 'broadcast.artist.content', 'Hot 108 Jamz'), ('1.FM Top 40', 'broadcast.content.artist', 'Delerium'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The BossHoss'), ('Teen idol', 'base.icons.icon_genre.icons', 'Annette Funicello'), ('Teen idol', 'common.topic.webpage', 'm.09wf9d1'), ('Jay-Z', 'broadcast.artist.content', 'HitzRadio.com'), ('Pattie Mallette', 'people.person.profession', 'Writer'), ('Michael Jackson', 'people.person.profession', 'Record producer'), (\"Justin Bieber's Believe\", 'film.film.country', 'United States of America'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Height'), ('Kelis', 'music.artist.genre', 'Dance music'), ('Twista', 'people.person.nationality', 'United States of America'), ('m.05sp405', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('Justin Timberlake', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('DMX', 'broadcast.artist.content', 'HitzRadio.com'), ('Demi Lovato', 'music.artist.genre', 'Contemporary R&B'), ('Gwen Stefani', 'music.artist.genre', 'Dance music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Frank Sinatra'), ('Heartbreaker', 'music.album.primary_release', 'Heartbreaker'), ('Jessie J', 'people.person.profession', 'Singer'), ('m.012nv3hv', 'business.employment_tenure.person', 'Yves Bole'), ('Janet Jackson', 'music.artist.genre', 'Rhythm and blues'), ('Pearl Jam', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('Snoop Dogg', 'people.person.languages', 'English Language'), ('m.0w5l5h1', 'education.education.institution', 'St. Michael Catholic Secondary School'), ('Elvis Presley', 'people.person.profession', 'Musician'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_73g3'), ('m.0y4tdml', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Nelly', 'people.person.profession', 'Record producer'), ('m.0njhx1b', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award category'), ('m.0101ftt8', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Kuk Harrell', 'music.artist.genre', 'Dance-pop'), ('Shaggy', 'common.topic.notable_types', 'Musical Artist'), ('Under the Mistletoe', 'music.album.release_type', 'Album'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Club Mix)'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Conjure One'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Profession'), ('#Thatpower', 'common.topic.notable_for', 'g.11b5m2tmhn'), ('Keyshia Cole', 'people.person.nationality', 'United States of America'), ('m.0zbf_4g', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Singer-songwriter', 'common.topic.subject_of', 'Bleona'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvc3'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4x'), ('Gavin DeGraw', 'broadcast.artist.content', '1.FM Top 40'), ('m.0_sxby0', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('As Long as You Love Me', 'music.album.artist', 'Justin Bieber'), ('1Club.FM: V101', 'broadcast.content.artist', 'Brandy Norwood'), ('1Club.FM: Power', 'broadcast.content.producer', '1Club.FM'), ('All That Matters', 'music.composition.composer', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'common.topic.notable_types', 'Musical Recording'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109j5xt'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('P!nk', 'music.artist.genre', 'Rhythm and blues'), ('Nicki Minaj', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('HitzRadio.com', 'broadcast.content.artist', 'Lady Gaga'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('First Dance', 'music.composition.composer', 'Dwight Reynolds'), ('m.0101ft1r', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Boyfriend', 'music.single.versions', 'Boyfriend (Dada Life remix)'), ('Pop music', 'music.genre.parent_genre', 'Rhythm and blues'), ('Beauty And A Beat', 'music.composition.composer', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jessie James Decker'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Montell Jordan', 'common.topic.notable_types', 'Musical Artist'), ('Foreign Remix', 'music.recording.tracks', 'Foreign Remix'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Blake Lewis'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0njhvsq'), (\"O'Kelly Isley, Jr.\", 'music.composer.compositions', 'Bad Day'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.05sp405'), ('Paul Anka', 'people.person.nationality', 'United States of America'), ('Nelly Furtado', 'music.artist.genre', 'Hip hop music'), ('Green Day', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0gbm3cg', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Nathan Lanier', 'people.person.gender', 'Male'), ('Jessie J', 'music.artist.label', 'Island Records'), ('Caitlin Beadles', 'base.popstra.celebrity.breakup', 'm.0gxnp26'), ('Yves Bole', 'people.person.places_lived', 'm.012nv5h3'), ('P!nk', 'music.artist.genre', 'Pop music'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Dapo Torimiro', 'music.artist.genre', 'Rock music'), ('Madonna', 'influence.influence_node.influenced', 'Lady Gaga'), ('Donna Summer', 'people.person.gender', 'Female'), ('Dance music', 'music.genre.subgenre', 'K-pop'), ('m.0sxhgqj', 'award.award_nomination.ceremony', 'Juno Awards of 2013'), ('Britney Spears', 'music.artist.genre', 'Teen pop'), ('m.0j2189_', 'music.music_video_performance.music_video_character', 'Singer'), ('Favorite Girl', 'music.album.releases', 'Favorite Girl'), ('Zendaya: Behind the Scenes', 'common.topic.notable_types', 'Film'), ('Roller Coaster', 'music.album.artist', 'Justin Bieber'), ('Blackstreet', 'music.artist.genre', 'Hip hop music'), ('m.09wxbsr', 'common.webpage.topic', 'Record producer'), ('m.0njvtth', 'award.award_honor.honored_for', 'My World 2.0'), ('London', 'location.location.containedby', 'Canada'), ('Never Let You Go', 'music.composition.lyricist', 'Justin Bieber'), ('m.0nh04xx', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ernie Isley', 'people.person.gender', 'Male'), ('Urban contemporary', 'broadcast.genre.content', '181-thebox'), ('Kid Cudi', 'people.person.languages', 'English Language'), ('Usher', 'people.person.profession', 'Music executive'), ('1Club.FM: V101', 'broadcast.content.genre', 'Classic hits'), ('Nick Jonas', 'people.person.profession', 'Singer-songwriter'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Gollum & Yanny'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('Gavin DeGraw', 'common.topic.notable_types', 'Musical Artist'), ('m.0kyjls4', 'music.music_video_performance.music_video_character', 'Singer'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful and the Beat', 'common.topic.notable_types', 'Musical Recording'), ('Kid Cudi', 'people.person.gender', 'Male'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Lil Jon'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Lil Wayne'), ('1.FM Top 40', 'broadcast.content.artist', 'Britney Spears'), ('Twista', 'broadcast.artist.content', '1.FM Top 40'), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.episode', \"Boot Camp No. 3 / Judge's House No. 1\"), ('Jason Mraz', 'music.artist.genre', 'Indie rock'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ray J', 'music.artist.genre', 'Rhythm and blues'), ('Smoothbeats', 'common.topic.notable_types', 'Broadcast Content'), ('Dapo Torimiro', 'music.artist.genre', 'Pop music'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Jay-Z'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.episode', 'Episode 3'), ('Sunshine Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Juicy J', 'people.person.gender', 'Male'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Chloe Bridges'), ('Recovery', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Jason McCann', 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr2dh'), ('m.011m26pv', 'music.group_membership.role', 'Vocals'), ('Pray', 'music.composition.form', 'Song'), ('Yves Bole', 'base.musicpf.artist.genre', 'Singer-songwriter'), ('Janet Jackson', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('R. Kelly', 'people.person.profession', 'Singer'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Usher'), ('m.0gbm3bl', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('The Notorious B.I.G.', 'people.person.profession', 'Singer'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'King Ku$ha'), ('m.0yrjynf', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('Thought Of You', 'music.recording.artist', 'Justin Bieber'), ('m.0njdq6k', 'award.award_honor.award_winner', 'Justin Bieber'), ('Miami', 'location.hud_county_place.place', 'Miami'), ('Bad Day', 'common.topic.notable_for', 'g.1yg4drkzk'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bm2h'), ('Akon', 'music.artist.genre', 'Reggae'), ('m.0njhx1b', 'award.award_honor.award_winner', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', \"Nino D'Angelo\"), ('Madonna', 'people.person.languages', 'English Language'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Kanye West', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Shaffer Smith', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.written_by', 'Sarah Landman'), ('My Worlds', 'common.topic.notable_types', 'Musical Album'), ('Bigger', 'common.topic.notable_for', 'g.1z25545x7'), ('Rock music', 'common.topic.subject_of', 'Alan Motley'), ('PowerHitz', 'broadcast.content.artist', 'King Ku$ha'), ('Keyshia Cole', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr2qn'), ('iJustine', 'people.person.profession', 'Actor'), ('Shaffer Smith', 'music.artist.genre', 'Europop'), ('Toby Gad', 'music.artist.genre', 'Dance music'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Kevin Risto', 'music.lyricist.lyrics_written', 'Bigger'), ('Die in Your Arms', 'music.album.releases', 'Die in Your Arms'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Nelly Furtado', 'people.person.profession', 'Singer-songwriter'), ('m.0gbm3cp', 'film.personal_film_appearance.person', 'Sean Kingston'), ('Date of birth', 'type.property.expected_type', 'Date/Time'), ('HitzRadio.com', 'broadcast.content.artist', 'Christina Milian'), ('Dance music', 'broadcast.genre.content', '1.FM Top 40'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Big Sean', 'music.artist.track', 'As Long As You Love Me'), ('Jaden Smith', 'people.person.profession', 'Actor'), ('Duffy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Estelle'), ('Diplo', 'music.artist.genre', 'Electronic music'), ('Jayceon Terrell Taylor', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.artist', 'Sean Paul'), ('Confident', 'music.album.artist', 'Justin Bieber'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Men in Black 3', 'common.topic.notable_types', 'Award-Winning Work'), ('m.0yrjvlh', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('My Worlds: The Collection', 'common.topic.image', 'Justin-Bieber-My-Worlds-The-Collection.jpg'), ('Record producer', 'common.topic.webpage', 'm.09wxbsr'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011vfnlz'), ('Christina Milian', 'people.person.profession', 'Singer-songwriter'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('MTV Europe Music Award for Best World Stage Performance', 'award.award_category.winners', 'm.0z340zt'), ('Billboard Music Award for Top Pop Album', 'award.award_category.nominees', 'm.0j8z6tl'), ('HitzRadio.com', 'broadcast.content.artist', 'Nickelback'), ('All Around the World', 'music.album.releases', 'All Around the World'), ('Never Let You Go', 'music.album.artist', 'Justin Bieber'), ('Chris Jasper', 'common.topic.notable_types', 'Musical Artist'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Craig David'), ('Nicki Minaj', 'music.artist.genre', 'Hip hop music'), ('Runaway Love (remix)', 'music.recording.featured_artists', 'Raekwon'), ('Justin Bieber', 'music.composer.compositions', 'Never Let You Go'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_70rd'), ('Jessie J', 'music.artist.genre', 'Dance-pop'), ('Nicki Minaj', 'music.artist.genre', 'Rhythm and blues'), ('m.0gbm3dn', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bad Day', 'common.topic.notable_types', 'Musical Recording'), ('Gwen Stefani', 'people.person.profession', 'Singer-songwriter'), ('Love Never Felt So Good', 'common.topic.notable_for', 'g.1s05bndft'), ('Big R Radio - The Hawk', 'broadcast.content.producer', 'Big R Radio Network'), ('Hot Wired Radio', 'broadcast.content.artist', 'Justin Timberlake'), ('Urban contemporary', 'broadcast.genre.content', 'Hot 97.7'), ('Jason Mraz', 'common.topic.notable_types', 'Musical Artist'), ('Ginuwine', 'broadcast.artist.content', 'HitzRadio.com'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0z898w6'), ('Lady Gaga', 'people.person.gender', 'Female'), ('Teen Choice Award for Choice Summer Music Star: Male', 'award.award_category.winners', 'm.0yrkgd6'), ('Right Here', 'common.topic.article', 'm.0qfpkqx'), ('Justin Bieber', 'music.featured_artist.recordings', 'Next to You'), ('Daniel Bedingfield', 'people.person.profession', 'Singer'), ('Fall Out Boy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('181-beat', 'broadcast.content.artist', 'Justin Timberlake'), ('As Long as You Love Me (acoustic version)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Award-Winning Work', 'type.type.expected_by', 'Winning work'), ('HitzRadio.com', 'broadcast.content.artist', 'Akon'), ('Blu Cantrell', 'people.person.profession', 'Musician'), ('Akon', 'music.artist.genre', 'Dance-pop'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hip hop music', 'broadcast.genre.content', '1Club.FM: Channel One'), ('Believe Tour', 'music.concert_tour.artist', 'Justin Bieber'), ('Anastacia', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'music.artist.album', 'Love Me'), ('As Long as You Love Me', 'music.composition.composer', 'Big Sean'), ('1.FM Top 40', 'broadcast.content.genre', 'Top 40'), ('Believe Acoustic', 'common.topic.article', 'm.0pb8twm'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Top 40'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('Don Henley', 'people.person.profession', 'Singer'), ('RedOne', 'music.artist.genre', 'Electronic dance music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Fray'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0tkqqgg'), ('Ronald Isley', 'freebase.valuenotation.has_value', 'Parents'), ('Actor', 'base.lightweight.profession.professions_similar', 'Model'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jayceon Terrell Taylor'), ('Ronald Isley', 'people.person.profession', 'Record producer'), ('Under the Mistletoe', 'common.topic.notable_types', 'Musical Album'), ('Justin Bieber', 'music.artist.genre', 'Contemporary R&B'), ('Amerie', 'people.person.profession', 'Model'), ('Lil Wayne', 'music.artist.genre', 'Hip hop music'), ('Jennifer Lopez', 'people.person.languages', 'English Language'), ('JoJo', 'music.artist.genre', 'Rhythm and blues'), ('Kylie Minogue', 'music.artist.genre', 'Rhythm and blues'), ('Boyfriend', 'common.topic.notable_types', 'Award-Winning Work'), ('Turn to You (Mother’s Day Dedication)', 'music.album.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Nelly Furtado'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc3_1'), ('Never Say Never: The Remixes', 'music.album.genre', 'Contemporary R&B'), ('Rob Thomas', 'people.person.nationality', 'United States of America'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftn8'), ('Beauty and a Beat (acoustic version)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('HitzRadio.com', 'broadcast.content.artist', 'Mariah Carey'), ('Trey Songz', 'people.person.profession', 'Singer-songwriter'), ('Terius Nash', 'music.artist.genre', 'Hip hop music'), ('All That Matters', 'common.topic.notable_for', 'g.1yfp37src'), ('m.0gxnp0c', 'base.popstra.dated.participant', 'Justin Bieber'), ('Beauty and a Beat', 'music.album.featured_artists', 'Nicki Minaj'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Monifah'), ('CSI: Crime Scene Investigation', 'tv.tv_program.country_of_origin', 'United States of America'), ('Justin Bieber Romania', 'common.resource.annotations', 'm.0cq990t'), ('Justin Bieber', 'base.schemastaging.person_extra.net_worth', 'm.0yqflhy'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Ricky Nelson', 'people.person.profession', 'Musician'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Official website'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Chingy'), ('Justin Bieber', 'music.artist.album', 'My Worlds: The Collection'), ('m.0z0tmyv', 'award.award_honor.ceremony', '4th Annual Shorty Awards'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Love Never Felt So Good', 'music.album.primary_release', 'Love Never Felt So Good'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Profession'), ('United States of America', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ludacris', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Children', 'type.property.master_property', 'Parents'), ('Annette Funicello', 'people.person.profession', 'Singer'), ('Ludacris', 'broadcast.artist.content', 'PowerHitz'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'music.composer.compositions', 'Wait for a Minute'), ('Selena Gomez', 'celebrities.celebrity.sexual_relationships', 'm.0gxnnzy'), ('Trick Daddy', 'broadcast.artist.content', '181-beat'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Teen idol', 'common.topic.webpage', 'm.09y8dts'), ('All That Matters (video)', 'common.topic.notable_types', 'Musical Recording'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Sean Combs'), ('Change Me', 'common.topic.notable_for', 'g.1yp3dqvz2'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Michael Jackson'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhxd_'), ('m.0sgkw_d', 'award.award_honor.award_winner', 'Justin Bieber'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Gender'), ('William Orbit', 'people.person.gender', 'Male'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Los Niños de Sara'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.article', 'm.0jwqwz0'), ('Somebody to Love (remix)', 'music.album.featured_artists', 'Usher'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Leona Lewis'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Will Smith'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Madonna'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Jessie J'), ('Justin Bieber', 'music.artist.album', 'Never Let You Go'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s_wn'), ('Teen pop', 'common.topic.notable_types', 'Musical genre'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Year'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Timbaland', 'music.group_member.artists_supported', 'Bleona'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Nelly', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('The Isley Brothers', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0sxhq3d'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Baby Bash'), ('Baby', 'music.composition.recordings', 'Baby'), ('Ludacris', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3d3'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjynf'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr2dh'), ('m.0102z0vx', 'award.award_honor.ceremony', 'Juno Awards of 2014'), ('m.0wjgqck', 'award.award_honor.award_winner', 'Justin Bieber'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien Luvstep mix)'), ('Never Say Never', 'common.topic.notable_for', 'g.1yl5lrh_1'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('Ashley Tisdale', 'music.artist.genre', 'Rhythm and blues'), ('Ellen DeGeneres', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft3d'), ('Pearl Jam', 'broadcast.artist.content', 'Hot Wired Radio'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Country of nationality', 'type.property.schema', 'Person'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kylie Minogue', 'music.artist.genre', 'Pop music'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronald Isley', 'people.person.profession', 'Singer'), ('Synthpop', 'music.genre.subgenre', 'Synthpop'), ('Chingy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('BeirutNights.com Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Jessica Simpson', 'people.person.gender', 'Female'), ('Singer-songwriter', 'people.profession.specialization_of', 'Singer'), ('m.0wjgrzc', 'award.award_honor.award_winner', 'Justin Bieber'), ('PowerHitz', 'broadcast.content.artist', 'Flo Rida'), ('HitzRadio.com', 'broadcast.content.artist', 'Buckcherry'), ('Record producer', 'common.topic.webpage', 'm.09wwfnk'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulClassics.com'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Alicia Keys'), ('Nelly Furtado', 'common.topic.notable_types', 'Musical Artist'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5kxx'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101ft2v', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Adrienne Bailon', 'music.artist.genre', 'Rhythm and blues'), ('Will i Am', 'people.person.profession', 'Dancer'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z1scxk'), ('Gavin DeGraw', 'broadcast.artist.content', 'HitzRadio.com'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Capone-N-Noreaga'), ('RedOne', 'music.artist.genre', 'Eurodance'), ('Hot Wired Radio', 'broadcast.content.artist', 'P!nk'), ('Roller Coaster', 'common.topic.notable_types', 'Musical Recording'), ('m.0101ftks', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Teen idol', 'common.topic.image', 'Rudolph Valentino'), ('m.0yrk4gn', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Savan Kotecha', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0gbm3d9', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Dan Cutforth', 'people.person.gender', 'Male'), ('Katy Perry', 'music.artist.genre', 'Teen pop'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Sean Combs', 'music.artist.genre', 'Pop music'), ('Hold Tight', 'music.composition.recordings', 'Hold Tight'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Kelly Clarkson', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'music.album.artist', 'Far East Movement'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('World music', 'broadcast.genre.content', 'BeirutNights.com Radio'), ('As Long as You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Nicki Minaj', 'people.person.profession', 'Singer-songwriter'), ('HitzRadio.com', 'broadcast.content.broadcast', 'HitzRadio.com - 128kbps Stream'), ('Max Martin', 'music.artist.genre', 'Europop'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.artist', 'Justin Bieber'), ('Jordan Francis', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Chris Jasper', 'music.composer.compositions', 'Bad Day'), ('Tricky Stewart', 'people.person.profession', 'Musician'), ('Somebody to Love', 'music.composition.composer', 'Heather Bright'), ('English Language', 'language.human_language.countries_spoken_in', 'Canada'), ('Music Producer', 'common.topic.notable_types', 'Profession'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Chris Brown', 'music.artist.genre', 'Dance music'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO Todays RNB'), ('181-beat', 'broadcast.content.genre', 'Contemporary R&B'), ('Brandy Norwood', 'people.person.profession', 'Actor'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Somebody to Love (J Stax remix)', 'music.recording.canonical_version', 'Somebody To Love'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Children'), ('m.0njvs9s', 'award.award_honor.award_winner', 'Justin Bieber'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_svs8m'), ('m.0gctps1', 'tv.tv_guest_role.episodes_appeared_in', 'Miranda Hart, Russell Howard, Justin Bieber, David Haye and Florence and the Machine'), ('Pray', 'music.recording.artist', 'Justin Bieber'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhyh_'), ('Jennifer Lopez', 'music.artist.label', 'Island Records'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw59_'), ('Kanye West', 'music.artist.genre', 'Pop music'), ('Kanye West', 'music.artist.genre', 'Rhythm and blues'), ('Somebody to Love (remix)', 'common.topic.notable_types', 'Musical Album'), ('Bad Day', 'music.album.artist', 'Justin Bieber'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.recording.song', 'Baby'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'M.I.A.'), ('Gwen Stefani', 'people.person.profession', 'Artist'), ('181-beat', 'broadcast.content.artist', 'Usher'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Official website'), ('m.012bm2cn', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Soulja Boy', 'people.person.gender', 'Male'), ('m.0yqflhy', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Artist'), ('m.07053l5', 'common.webpage.topic', 'Justin Bieber'), ('m.0y7y53r', 'award.award_honor.ceremony', 'Kerrang! Awards 2012'), ('Kelly Clarkson', 'music.artist.genre', 'Contemporary R&B'), ('Dapo Torimiro', 'music.composer.compositions', 'Bigger'), ('m.0gbm3bl', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('The Island Def Jam Music Group', 'organization.organization.previous_names', 'm.05sp41b'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Classic hits'), ('Rita Ora', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'), ('Lolly', 'music.album.primary_release', 'Lolly'), ('Marvin Isley', 'people.person.nationality', 'United States of America'), ('The Black Eyed Peas', 'music.artist.genre', 'Dance music'), ('m.0z8755b', 'award.award_honor.ceremony', 'NME Awards 2012'), ('Miley Cyrus', 'people.person.profession', 'Dancer'), ('Justin Bieber: Just Getting Started', 'book.written_work.original_language', 'English Language'), ('First Dance', 'music.composition.composer', 'Usher'), ('As Long As You Love Me (Audiobot instrumental)', 'common.topic.notable_types', 'Musical Recording'), ('Zac Brown Band', 'common.topic.notable_types', 'Musical Artist'), ('Donna Summer', 'music.artist.origin', 'United States of America'), ('Jennifer Lopez', 'music.artist.genre', 'Pop music'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('m.0cq990t', 'common.webpage.topic', 'Justin Bieber'), ('Bad 25', 'film.film.language', 'English Language'), ('Shaggy', 'freebase.valuenotation.has_value', 'Parents'), ('Eminem', 'common.topic.notable_types', 'Musical Artist'), ('Hit-Boy', 'common.topic.notable_types', 'Musical Artist'), ('Beautiful', 'common.topic.notable_types', 'Musical Recording'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.episode', 'Series 1 Episode 18'), ('Beyoncé Knowles', 'music.artist.genre', 'Electronic dance music'), ('Baby', 'music.album.compositions', 'Baby'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Rita Ora', 'music.artist.genre', 'Rhythm and blues'), ('Hit-Boy', 'people.person.profession', 'Record producer'), ('Change Me', 'common.topic.notable_types', 'Musical Album'), ('Avril Lavigne', 'music.artist.genre', 'Pop music'), ('Boyfriend', 'common.topic.notable_for', 'g.1259shy1p'), ('Daniel Bedingfield', 'music.artist.label', 'The Island Def Jam Music Group'), ('Live My Life', 'common.topic.notable_types', 'Musical Album'), ('Estelle', 'common.topic.notable_types', 'Musical Artist'), ('Music', 'common.topic.subjects', 'Artfarm'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ashley Tisdale'), ('m.0njhvsq', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'Randy Phillips'), ('Shaun Cassidy', 'people.person.profession', 'Actor'), ('Jason McCann', 'tv.tv_character.appeared_in_tv_episodes', 'm.0pcr28j'), ('Contemporary R&B', 'broadcast.genre.content', 'JellyRadio.com'), ('radioIO RNB Mix', 'broadcast.content.genre', \"00's\"), ('Redfoo', 'people.person.profession', 'Dancer'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: V101', 'broadcast.content.artist', 'Santana'), ('Yves Bole', 'people.person.profession', 'Model'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Jason Mraz'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0zbf_4g'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjhb'), ('Rihanna', 'broadcast.artist.content', 'radioIO Todays POP'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rodney Jerkins', 'people.person.gender', 'Male'), ('Montell Jordan', 'people.person.nationality', 'United States of America'), ('Duffy', 'music.artist.genre', 'Rhythm and blues'), ('181-beat', 'broadcast.content.artist', 'Twista'), ('Camagüey', 'location.location.people_born_here', 'Yves Bole'), ('PowerHitz', 'broadcast.content.artist', 'Chris Brown'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kid Cudi'), ('Kings of Leon', 'broadcast.artist.content', 'Hot Wired Radio'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rob Thomas'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber', 'base.popstra.celebrity.dated', 'm.0gxnp02'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Buckcherry'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Country of nationality'), ('2011 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0115qhzk'), ('Recovery', 'music.album.artist', 'Justin Bieber'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Author', 'freebase.type_hints.included_types', 'Topic'), ('Rudolph Isley', 'freebase.valuenotation.has_value', 'Parents'), ('My Worlds: The Collection', 'music.album.artist', 'Justin Bieber'), ('Avery', 'music.artist.genre', 'Pop music'), ('Nelly Furtado', 'music.artist.genre', 'World music'), ('m.0yrkc0l', 'award.award_honor.award_winner', 'Justin Bieber'), ('Sheryl Crow', 'broadcast.artist.content', 'HitzRadio.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Fergie'), ('Keyshia Cole', 'people.person.profession', 'Singer'), ('Justin Bieber', 'music.artist.album', 'Somebody to Love (remix)'), ('August Rigo', 'music.artist.genre', 'Pop music'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('radioIO Classic RNB', 'common.topic.notable_types', 'Broadcast Content'), ('The Black Eyed Peas', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkw8v'), ('Beauty and a Beat (DJ Laszlo Body Rock Instrumental)', 'music.recording.artist', 'Justin Bieber'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Urban contemporary'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten remix)'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z2dr9y'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('m.0_x3kvk', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Avril Lavigne', 'broadcast.artist.content', 'radioIO Todays POP'), ('Will Smith', 'music.artist.genre', 'Urban contemporary'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('CL', 'music.artist.genre', 'Contemporary R&B'), ('m.012r2w0k', 'celebrities.friendship.friend', 'Athan Grace'), ('Demi Lovato', 'music.artist.genre', 'Rhythm and blues'), ('Billboard Music Award for Top Streaming Song (Video)', 'award.award_category.winners', 'm.0njhyh_'), ('Somebody to Love', 'common.topic.notable_for', 'g.12h2wlms9'), ('Baby', 'music.album.genre', 'Electronic music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvvj_'), ('Usher', 'people.person.profession', 'Film Producer'), ('All Around the World', 'music.single.versions', 'All Around the World (acoustic version)'), ('Chris Brown', 'people.person.languages', 'English Language'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Classic hits'), ('m.0z8s562', 'award.award_honor.award_winner', 'Justin Bieber'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Contemporary R&B'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Günther & The Sunshine Girls'), ('My World', 'music.album.releases', 'My World'), ('m.0rqp4h0', 'music.track_contribution.contributor', 'Justin Bieber'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Height'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsk'), ('Raekwon', 'people.person.gender', 'Male'), ('HitzRadio.com', 'broadcast.content.artist', 'Dr. Dre'), ('m.0njvtth', 'award.award_honor.award_winner', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Ludacris'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c92p'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Akon'), ('Red Hot Chili Peppers', 'common.topic.notable_types', 'Musical Artist'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Island Def Jam Music Group', 'common.topic.webpage', 'm.064xzb1'), ('Savan Kotecha', 'music.artist.genre', 'Teen pop'), ('Believe', 'music.album.release_type', 'Album'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.tracks', 'Wont Stop (feat. Justin Bieber)'), ('Spouse (or domestic partner)', 'rdf-schema#range', 'Marriage'), ('PowerHitz', 'broadcast.content.artist', 'Juvenile'), ('Khalil', 'music.artist.genre', 'Contemporary R&B'), ('radioIO Todays RNB', 'broadcast.content.location', 'Tampa'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('Justin Bieber', 'music.artist.album', 'My Worlds Acoustic'), ('Cris Cab', 'people.person.gender', 'Male'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bq275'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Phoenix'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('Scooter Braun', 'people.person.nationality', 'United States of America'), ('Runaway Love (remix)', 'common.topic.notable_for', 'g.1yg9g44l8'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Johnny Crawford', 'people.person.profession', 'Singer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Rap music'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (acoustic version)'), ('Bryan Adams', 'people.person.profession', 'Actor'), ('Nasri', 'people.person.place_of_birth', 'Canada'), ('Daniel Bedingfield', 'music.artist.genre', 'Electronic music'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Chingy'), ('Heartbreaker', 'music.album.compositions', 'Heartbreaker'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0pcnqnb'), ('Donna Summer', 'music.artist.genre', 'Rhythm and blues'), ('Ricky Nelson', 'people.person.nationality', 'United States of America'), ('1Club.FM: Channel One', 'broadcast.content.producer', '1Club.FM'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Shaggy'), ('Justin Bieber', 'broadcast.artist.content', '.977 The Hits Channel'), ('Never Say Never: The Remixes', 'music.album.album_content_type', 'Remix album'), ('HitzRadio.com', 'broadcast.content.artist', 'Mary J. Blige'), ('Miley Cyrus', 'broadcast.artist.content', 'radioIO Todays POP'), ('#thatPOWER', 'music.single.versions', '#Thatpower'), ('Usher', 'broadcast.artist.content', 'FLOW 103'), ('Beauty and a Beast', 'common.topic.notable_types', 'Musical Recording'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mary J. Blige'), ('1Club.FM: Power', 'broadcast.content.artist', 'Ja Rule'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life (Party Rock remix)'), ('American Music Awards of 2010', 'award.award_ceremony.nominees', 'm.0zbg2kw'), ('Shaffer Smith', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Lupe Fiasco', 'people.person.profession', 'Singer'), ('Gwen Stefani', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Synthpop', 'music.genre.parent_genre', 'Rock music'), ('Sheryl Crow', 'people.person.nationality', 'United States of America'), ('Demi Lovato', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('Alicia Keys', 'broadcast.artist.content', 'FLOW 103'), ('Gwen Stefani', 'people.person.profession', 'Actor'), ('Ontario', 'location.administrative_division.first_level_division_of', 'Canada'), ('The Isley Brothers', 'music.artist.label', 'Island Records'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Next'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Year'), ('1.FM Top 40', 'broadcast.content.artist', 'Puddle of Mudd'), ('All That Matters (video)', 'music.recording.song', 'All That Matters'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Tommy Sands', 'people.person.place_of_birth', 'Chicago'), ('Justin bieber', 'common.image.appears_in_topic_gallery', 'Justin Bieber'), ('m.0yqfny6', 'award.award_honor.award', 'MTV Europe Music Award for Best Canadian Act'), ('Usher', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Redfoo', 'common.topic.notable_types', 'Musical Artist'), ('School Boy Records', 'organization.organization.founders', 'Scooter Braun'), ('Disney Parks Christmas Day Parade', 'film.film.genre', 'Music'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv97'), ('United States Dollar', 'finance.currency.countries_used', 'United States of America'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9hp2'), ('Alicia Keys', 'people.person.profession', 'Musician'), ('Whitney Houston', 'music.artist.genre', 'Rhythm and blues'), ('Victoria Justice', 'music.artist.genre', 'Dance-pop'), ('Contemporary R&B', 'common.topic.article', 'm.025sc53'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0yrk0mt', 'award.award_honor.award_winner', 'Justin Bieber'), ('GotRadio - RnB Classics', 'broadcast.content.genre', 'Contemporary R&B'), ('50 Cent', 'broadcast.artist.content', 'FLOW 103'), ('All Around the World (acoustic version)', 'music.recording.canonical_version', 'All Around the World'), ('Jazmyn Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Katy Perry', 'people.person.gender', 'Female'), ('Dr. Dre', 'people.person.profession', 'Actor'), ('Trick Daddy', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0ywtfj6', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Official website', 'type.property.schema', 'Topic'), ('Beyoncé Knowles', 'broadcast.artist.content', 'HitzRadio.com'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Shiny Toy Guns'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Believe', 'music.album.supporting_tours', 'Believe Tour'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Nasri', 'music.composer.compositions', 'Pray'), ('Trick Daddy', 'people.person.place_of_birth', 'Miami'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvkl'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Everlast'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Jordin Sparks', 'broadcast.artist.content', '.977 The Hits Channel'), ('1.FM Top 40', 'broadcast.content.artist', 'U2'), ('m.0z0tmyv', 'award.award_honor.award', 'Shorty Award for Music'), ('Jennifer Lopez', 'people.person.profession', 'Record producer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Jay-Z'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Cascada'), ('Athan Grace', 'people.person.gender', 'Male'), ('m.0zg2w8r', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Cassie Ventura'), ('Reed Smoot', 'film.cinematographer.film', 'Justin Bieber: Never Say Never'), ('m.0yrjynf', 'award.award_honor.award_winner', 'Justin Bieber'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6_ww'), ('Little Bird', 'music.recording.artist', 'Far East Movement'), ('Scooter Braun', 'organization.organization_founder.organizations_founded', 'School Boy Records'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Calling'), ('1Club.FM: V101', 'broadcast.content.producer', '1Club.FM'), ('New Kids on the Block', 'broadcast.artist.content', '1Club.FM: Channel One'), ('1Club.FM: V101', 'broadcast.content.artist', 'Leona Lewis'), ('Geri Halliwell', 'people.person.gender', 'Female'), ('m.0yrk0mt', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Christina Aguilera', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.0101ftww', 'film.film_cut.film', \"Justin Bieber's Believe\"), ('Raekwon', 'music.featured_artist.albums', 'Runaway Love (remix)'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Parents'), ('All That Matters (video)', 'music.recording.canonical_version', 'All That Matters'), ('Recovery', 'music.album.releases', 'Recovery'), ('Jennifer Lopez', 'broadcast.artist.content', 'Sunshine Radio'), ('Never Say Never: The Remixes', 'music.album.genre', 'Teen pop'), ('Montell Jordan', 'music.artist.genre', 'Contemporary R&B'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njgyk4'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Terius Nash', 'common.topic.notable_types', 'Musical Artist'), ('Urban contemporary', 'broadcast.genre.content', 'FLOW 103'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_3hn'), ('HitzRadio.com', 'broadcast.content.artist', 'Sheryl Crow'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Parents'), ('Aaliyah', 'broadcast.artist.content', 'radioIO RNB Mix'), ('m.0nh04xx', 'film.personal_film_appearance.film', 'Justin Bieber: Rise to Fame'), ('P!nk', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('m.011m26pv', 'music.group_membership.member', 'Justin Bieber'), ('Gwen Stefani', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Gender'), ('UR Fave: New Artist', 'award.award_category.winners', 'm.0njwb81'), ('m.0pcr2qn', 'tv.tv_guest_role.episodes_appeared_in', 'Episode #2.10'), ('Jason Mraz', 'people.person.languages', 'English Language'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Parents'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Hip hop music'), ('Lady Gaga', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jennifer Lopez', 'music.artist.genre', 'Rhythm and blues'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Outkast'), ('Mariah Carey', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('HitzRadio.com', 'common.topic.article', 'm.03hs0fv'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Don Henley', 'people.person.profession', 'Musician'), ('Barry Weiss', 'music.artist.genre', 'Pop music'), ('Contemporary R&B', 'broadcast.genre.content', 'C9 Radio'), ('Sheryl Crow', 'broadcast.artist.content', 'Sunshine Radio'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ciara'), ('R. Kelly', 'music.artist.genre', 'Hip hop music'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Pop music'), ('Timbaland', 'people.person.gender', 'Male'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gts1'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Geri Halliwell', 'music.artist.genre', 'Synthpop'), ('Athan Grace', 'people.person.gender', 'Male'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jordin Sparks'), ('SoulfulClassics.com', 'common.topic.notable_types', 'Broadcast Content'), ('Ronski Speed', 'people.person.gender', 'Male'), ('Shaggy', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mann'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('m.0y4tdml', 'award.award_honor.ceremony', '2013 Radio Disney Music Awards'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Johnny Crawford', 'people.person.profession', 'Actor'), ('Stuart Ford', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ciara', 'broadcast.artist.content', 'PowerHitz'), ('1.FM Top 40', 'broadcast.content.artist', 'Rascal Flatts'), ('World music', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.featured_artists', 'Big Sean'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.featured_artists', 'Big Sean'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Children'), ('Yves Bole', 'base.musicpf.artist.album', 'Raindrops'), ('Miami Beach', 'location.location.time_zones', 'Eastern Time Zone'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012bq275'), ('Teyana', 'people.person.profession', 'Actor'), ('Wait For a Minute', 'music.album.artist', 'Tyga'), ('Rudolph Valentino', 'people.person.gender', 'Male'), ('m.0njwb81', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Usher', 'music.artist.genre', 'Pop music'), ('Hit-Boy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('My Worlds', 'music.album.primary_release', 'My Worlds'), ('Justin Bieber', 'music.artist.album', 'Confident'), ('Singer-songwriter', 'people.profession.part_of_professional_field', 'Singing'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('The Pussycat Dolls', 'broadcast.artist.content', 'FLOW 103'), ('As Long as You Love Me', 'music.album.release_type', 'Single'), ('Believe', 'common.topic.notable_types', 'Musical Album'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Ludacris'), ('Pray', 'music.composition.recordings', 'Pray'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Musical Album', 'type.type.properties', 'Initial release date'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('Gavin DeGraw', 'music.artist.genre', 'Acoustic music'), ('Shaggy', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Big Pun'), ('Teyana', 'common.topic.notable_types', 'Musical Artist'), ('Ronski Speed', 'common.topic.notable_types', 'Musical Artist'), ('Beyoncé Knowles', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Nelly', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0vpgdpb', 'music.recording_contribution.album', '#thatPOWER'), ('Duffy', 'people.person.profession', 'Actor'), ('#thatPower', 'music.composition.recordings', '#thatPower'), ('Timbaland', 'people.person.profession', 'Singer'), ('Roller Coaster', 'music.album.releases', 'Roller Coaster'), ('Under the Mistletoe', 'freebase.valuenotation.is_reviewed', 'Artist'), ('m.0pcr28j', 'tv.tv_guest_role.character', 'Jason McCann'), ('m.0101fvcp', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('PYD', 'award.award_nominated_work.award_nominations', 'm.0_x3kvk'), ('Mistletoe', 'music.album.compositions', 'Mistletoe'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'JoJo'), ('Enrique Iglesias', 'freebase.valuenotation.has_no_value', 'Children'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('Jon M. Chu', 'people.person.profession', 'Actor'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw257'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9hvh'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d3'), ('m.0njhxzc', 'award.award_honor.award', 'Billboard Music Award for Top Pop Album'), ('Right Here', 'music.single.versions', 'Right Here'), ('Height', 'rdf-schema#domain', 'Person'), ('Cory Gunz', 'common.topic.notable_types', 'Musical Artist'), ('Frank Sinatra', 'base.icons.icon.icon_genre', 'Teen idol'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Dance music'), ('Cris Cab', 'music.artist.genre', 'Pop music'), ('Singer', 'people.profession.specializations', 'Child singer'), ('m.0v_721l', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('m.0101ft0d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Hot 108 Jamz', 'broadcast.content.artist', 'Rihanna'), ('David Cassidy', 'people.person.profession', 'Singer'), ('First Dance', 'music.recording.song', 'First Dance'), ('m.0101fvp7', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Baby', 'music.album.release_type', 'Single'), ('Live My Life', 'music.composition.recordings', 'Little Bird'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Singer', 'people.profession.specializations', 'Sufi singer'), ('As Long as You Love Me', 'music.composition.composer', 'Rodney Jerkins'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Ciara'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Stephen Melton', 'common.topic.subjects', 'Rock music'), (\"O'Kelly Isley, Jr.\", 'people.person.gender', 'Male'), ('Geri Halliwell', 'people.person.profession', 'Singer-songwriter'), ('m.0101fvk9', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0t4s_bn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Reed Smoot', 'people.person.place_of_birth', 'United States of America'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Nelly Furtado'), ('C1', 'music.artist.label', 'The Island Def Jam Music Group'), ('Usher', 'music.lyricist.lyrics_written', 'First Dance'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gz0fr6'), ('Bad Day', 'music.composition.composer', 'Ernie Isley'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fszs'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0tjyljn', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Tricky Stewart', 'music.producer.releases_produced', 'My World'), ('Ludacris', 'people.person.profession', 'Film Producer'), ('Heartbreaker', 'music.composition.composer', 'Justin Bieber'), ('Christina Milian', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft1r'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.winners', 'm.0wjgqck'), ('Bryan Adams', 'broadcast.artist.content', '1.FM Top 40'), ('School Boy Records', 'common.topic.notable_types', 'Record label'), ('Will i Am', 'people.person.gender', 'Male'), ('m.0yrhmll', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0pc670l'), ('1Club.FM: V101', 'broadcast.content.artist', 'Chris Brown'), ('Madonna', 'music.artist.genre', 'Dance-pop'), (\"O'Kelly Isley, Jr.\", 'people.person.nationality', 'United States of America'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Clay Aiken', 'people.person.profession', 'Singer'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Jazmyn Bieber', 'people.person.gender', 'Female'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cute Is What We Aim For'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Right Here'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Frank Ocean'), ('#Thatpower', 'music.recording.featured_artists', 'Justin Bieber'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.04kjw8n'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Alice Deejay'), ('J. Holiday', 'music.artist.genre', 'Pop music'), ('Change Me', 'music.recording.artist', 'Justin Bieber'), ('RBMG Records', 'common.topic.notable_for', 'g.12579f_0q'), ('m.0gz0fr6', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('radioIO RNB Mix', 'broadcast.content.genre', 'Rhythm and blues'), ('Reggae', 'broadcast.genre.content', 'JellyRadio.com'), ('Change Me', 'freebase.valuenotation.is_reviewed', 'Composer'), ('1.FM Top 40', 'broadcast.content.artist', 'Christina Aguilera'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Chris Brown'), (\"2012 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkw_d'), ('Fall Out Boy', 'music.artist.label', 'Island Records'), ('m.0_x3kvk', 'award.award_nomination.nominated_for', 'PYD'), ('Ludacris', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0_cz5l3', 'celebrities.legal_entanglement.location', 'Miami Beach'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Marvin Isley', 'people.deceased_person.place_of_death', 'Chicago'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Judy Garland', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fvps', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Zac Efron', 'people.person.profession', 'Singer'), ('Justin Bieber', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Pras'), ('FLOW 103', 'broadcast.content.genre', 'Hip hop music'), ('m.0gctytd', 'tv.tv_guest_role.episodes_appeared_in', 'January 15, 2010'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('United States of America', 'common.topic.notable_types', 'Country'), ('m.0pcnqnb', 'film.personal_film_appearance.film', \"Dick Clark's Primetime New Year's Rockin' Eve 2013\"), ('Stratford Northwestern Secondary School', 'education.educational_institution.students_graduates', 'm.0yrlqp1'), ('Jennifer Lopez', 'people.person.gender', 'Female'), ('Nick Jonas', 'people.person.nationality', 'United States of America'), ('Beautiful and the Beat', 'music.recording.song', 'Beauty And A Beat'), ('Nicki Minaj', 'music.artist.genre', 'Pop music'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Yves Bole', 'influence.influence_node.influenced', 'Rihanna'), ('m.0njw1tn', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Kelly Clarkson'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfpbq5'), ('m.0yr9c1k', 'award.award_honor.honored_for', 'My World'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.010lkp2z', 'award.award_honor.ceremony', '2014 Billboard Music Awards'), ('Estelle', 'music.artist.genre', 'Electronic music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Estelle', 'music.artist.genre', 'Contemporary R&B'), ('Believe Acoustic', 'common.topic.notable_types', 'Musical Album'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Burnham'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('1Club.FM: Channel One', 'common.topic.image', '1clubfm.jpg'), ('All Around The World', 'award.award_nominated_work.award_nominations', 'm.0z85n_1'), ('Little Bird', 'music.recording.song', 'Live My Life'), ('Singer-songwriter', 'common.topic.subject_of', 'Stephen Melton'), ('Ashley Tisdale', 'freebase.valuenotation.is_reviewed', 'Gender'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ajda Pekkan'), ('Rihanna', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.artist', 'Justin Bieber'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Donna Summer', 'music.artist.genre', 'Dance-pop'), ('Donna Summer', 'common.topic.notable_types', 'Musical Artist'), ('m.012bm4v7', 'celebrities.friendship.friend', 'Yves Bole'), ('Under the Mistletoe', 'music.album.primary_release', 'Under the Mistletoe'), ('Don Henley', 'music.artist.genre', 'Pop music'), ('LeAnn Rimes', 'people.person.nationality', 'United States of America'), ('m.0ywvh8k', 'award.award_honor.ceremony', '5th Annual Shorty Awards'), ('Tyga', 'music.artist.genre', 'Hip hop music'), ('m.0yr9c1k', 'award.award_honor.award_winner', 'Justin Bieber'), ('Wait for a Minute', 'award.award_nominated_work.award_nominations', 'm.0_xlw5f'), ('Recovery', 'music.album.release_type', 'Single'), ('Lolly', 'music.album.artist', 'Maejor'), ('Rob Thomas', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)'), ('Justin Bieber: Rise to Fame', 'film.film.language', 'English Language'), ('Musical Album', 'type.type.properties', 'Release type'), ('Terence Dudley', 'common.topic.notable_types', 'Record Producer'), ('Where Are Ü Now', 'music.recording.featured_artists', 'Justin Bieber'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Flyleaf'), ('m.0njw444', 'award.award_honor.award_winner', 'Justin Bieber'), ('Clay Aiken', 'people.person.profession', 'Singer-songwriter'), (\"Turn to You (Mother's Day Dedication)\", 'music.recording.artist', 'Justin Bieber'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.artist', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Maurice Steenbergen'), ('JellyRadio.com', 'broadcast.content.artist', 'Tupac Shakur'), ('m.0z1scxk', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Shaggy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Mariah Carey', 'broadcast.artist.content', '1.FM Top 40'), ('Hip hop music', 'broadcast.genre.content', '181-thebox'), ('R. Kelly', 'common.topic.notable_types', 'Musical Artist'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njgyk4'), ('Sean Combs', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Deborah Lurie', 'people.person.nationality', 'United States of America'), ('Chris Brown', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Red Jumpsuit Apparatus'), ('Victoria Justice', 'people.person.profession', 'Dancer'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Rock music', 'common.topic.notable_types', 'Musical genre'), ('Where Are Ü Now', 'music.recording.tracks', 'g.11bv1nyzls'), ('Hit-Boy', 'music.composer.compositions', 'Right Here'), ('All Bad', 'common.topic.notable_for', 'g.1yn5bwn26'), ('Verse Simmonds', 'people.person.profession', 'Singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Justin Timberlake'), ('Cris Cab', 'people.person.profession', 'Musician'), ('m.09y8bsk', 'common.webpage.topic', 'Teen idol'), ('PowerHitz', 'broadcast.content.artist', 'Akon'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sisqó'), ('Soulja Boy', 'music.artist.genre', 'Dance music'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Blackstreet', 'music.artist.genre', 'Contemporary R&B'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0101ft5f', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('All Around The World', 'music.composition.composer', 'Nasri'), ('Will Smith', 'people.person.gender', 'Male'), ('Leif Garrett', 'people.person.profession', 'Actor'), ('m.0gbm3d_', 'film.personal_film_appearance.person', 'Jaden Smith'), ('Enrique Iglesias', 'people.person.profession', 'Actor'), ('Usher', 'people.person.profession', 'Dancer'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98_q'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Parents'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Colbie Caillat', 'people.person.profession', 'Singer-songwriter'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.featured_artists', 'Big Sean'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Record producer', 'common.topic.webpage', 'm.09wjs_l'), ('m.09wmk1t', 'common.webpage.topic', 'Teen pop'), ('m.0gxnp72', 'base.popstra.support.supporter', 'Justin Bieber'), ('Coldplay', 'broadcast.artist.content', '.977 The Hits Channel'), ('As Long As You Love Me (Audien dubstep mix)', 'common.topic.notable_types', 'Musical Recording'), ('Baby', 'music.composition.recordings', 'Baby'), ('#thatPOWER', 'common.topic.notable_for', 'g.12mq3wvxs'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0v90p2b'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Album'), ('Jason Mraz', 'people.person.profession', 'Musician'), ('Sean Combs', 'people.person.profession', 'Record producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Lady Sovereign'), ('Yves Bole', 'common.topic.notable_types', 'Musical Artist'), ('Kelly Clarkson', 'music.artist.genre', 'Rock music'), ('Die in Your Arms', 'common.topic.notable_for', 'g.126skfrp3'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fszb'), ('Ronald Isley', 'people.person.profession', 'Singer-songwriter'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Rock music'), ('m.0101fvkl', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Contemporary R&B', 'broadcast.genre.content', 'DeeGay'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'DMX'), ('Record producer', 'base.schemastaging.context_name.pronunciation', 'g.125_lq82d'), ('Rob Thomas', 'broadcast.artist.content', 'radioIO Todays POP'), ('#Thatpower', 'music.recording.song', '#thatPower'), ('HitzRadio.com', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Vanessa Hudgens', 'music.artist.genre', 'Contemporary R&B'), ('m.0_grqb8', 'celebrities.friendship.friend', 'Taylor Swift'), ('Somebody to Love', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'Recovery'), ('Stephen Melton', 'common.topic.subjects', 'Musician'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'award.award_winning_work.awards_won', 'm.0sgk_cw'), ('Jay-Z', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Fall Out Boy'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vz9'), ('Juelz Santana', 'people.person.nationality', 'United States of America'), ('2012 Billboard Music Awards', 'common.topic.notable_types', 'Award-Winning Work'), ('Colbie Caillat', 'people.person.profession', 'Artist'), ('m.0zbg2kw', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0gbm3c3', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Justin Bieber'), ('Beauty and a Beast', 'music.recording.song', 'Beauty And A Beat'), ('Parents', 'owl#inverseOf', 'Children'), ('PYD', 'music.composition.recordings', 'PYD'), ('Chicago', 'base.biblioness.bibs_location.country', 'United States of America'), ('m.0v_99fs', 'tv.tv_guest_personal_appearance.appearance_type', 'Him/Herself'), ('Hit-Boy', 'people.person.nationality', 'United States of America'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'DJ Cyglas vs. DJ Pure'), ('JoJo', 'broadcast.artist.content', 'radioIO Todays POP'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('radioIO Classic RNB', 'broadcast.content.location', 'Tampa'), ('Justin Timberlake', 'people.person.nationality', 'United States of America'), ('Roller Coaster', 'music.album.primary_release', 'Roller Coaster'), ('Believe', 'common.topic.notable_for', 'g.1259h0wh_'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Pop music'), ('All Around the World', 'common.topic.notable_types', 'Musical Album'), ('m.0gfpb_k', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('m.0sxhq3d', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien dubstep mix)'), ('m.0tk76mf', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY club mix)'), ('#Thatpower', 'common.topic.notable_types', 'Musical Recording'), ('Never Say Never', 'music.recording.artist', 'Justin Bieber'), ('Lupe Fiasco', 'people.person.profession', 'Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Top 40'), ('Justin Bieber: Just Getting Started', 'book.book.editions', 'Justin Bieber: Just Getting Started'), ('My Worlds: The Collection', 'common.topic.notable_for', 'g.125cvdbq7'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0r90dwg', 'common.webpage.resource', 'Justin Bieber Concerts'), ('L.A. Reid', 'music.artist.label', 'The Island Def Jam Music Group'), ('David Cassidy', 'people.person.nationality', 'United States of America'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsr'), ('Kings of Leon', 'music.artist.genre', 'Rock music'), ('PowerHitz', 'broadcast.content.artist', 'Shaffer Smith'), ('Johntá Austin', 'people.person.profession', 'Singer-songwriter'), ('Boyfriend', 'common.topic.notable_for', 'g.12590k9tt'), ('1Club.FM: Power', 'broadcast.content.artist', 'Shaggy'), ('Singer', 'people.profession.specializations', 'Ghost singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Radiohead'), ('Adam Messinger', 'music.artist.genre', 'Rock music'), ('Lolly', 'common.topic.notable_for', 'g.1yg94clyx'), ('m.0nh04xx', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beauty and a Beat (Remixes)', 'common.topic.notable_types', 'Musical Album'), ('Recovery', 'music.composition.recordings', 'Recovery'), ('As Long as You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaun Cassidy', 'people.person.profession', 'Singer'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Profession'), ('HitzRadio.com', 'broadcast.content.artist', 'Shaffer Smith'), ('Yves Bole', 'people.person.place_of_birth', 'Camagüey'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Linkin Park', 'broadcast.artist.content', 'radioIO Todays POP'), ('Mistletoe', 'common.topic.notable_for', 'g.12h2xc_71'), ('Yves Bole', 'influence.influence_node.influenced_by', 'Justin Bieber'), ('Hot 108 Jamz', 'common.topic.notable_types', 'Broadcast Content'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Black Eyed Peas'), ('Adrienne Bailon', 'people.person.gender', 'Female'), ('HitzRadio.com', 'broadcast.content.artist', 'Terius Nash'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Twista'), ('Miami', 'location.location.time_zones', 'Eastern Time Zone'), ('m.0z83x9l', 'award.award_honor.ceremony', 'NME Awards 2012'), ('Jordan Francis', 'music.artist.genre', 'Dance-pop'), ('Kerrang! Villain of the Year Award', 'award.award_category.winners', 'm.0y7y53r'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Official website'), ('English Language', 'language.human_language.main_country', 'United States of America'), ('Chris Jasper', 'music.artist.genre', 'Rhythm and blues'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw444'), ('m.0zbf_4g', 'award.award_honor.award_winner', 'Justin Bieber'), ('Kelly Clarkson', 'music.artist.genre', 'Pop music'), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_72tb'), ('m.0z84kwx', 'award.award_nomination.award', 'Teen Choice Award for Choice Single: Male Artist'), ('m.0vmyv4w', 'music.recording_contribution.contributor', 'Ludacris'), ('Sheryl Crow', 'people.person.gender', 'Female'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The All-American Rejects'), ('The Roots', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('m.0yrktlv', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Runaway Love', 'music.single.versions', 'Runaway Love (remix)'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO Todays POP'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0k05qcr'), ('Whitney Houston', 'people.person.profession', 'Artist'), ('Fergie', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3bs'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Children'), ('Journals', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Ellen DeGeneres', 'people.person.profession', 'Film Producer'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Jordin Sparks'), ('m.0sgkyfg', 'award.award_honor.ceremony', \"2011 Kids' Choice Awards\"), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Pussycat Dolls'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Milk & Sugar'), ('m.0d33gyj', 'celebrities.romantic_relationship.celebrity', 'Caitlin Beadles'), ('m.0w5l5h1', 'freebase.valuenotation.has_value', 'Start Date'), ('Usher', 'music.featured_artist.albums', 'Somebody to Love (remix)'), ('Nicki Minaj', 'freebase.valuenotation.has_no_value', 'Children'), ('m.0wjgrzc', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('m.0d33gyj', 'celebrities.romantic_relationship.celebrity', 'Justin Bieber'), ('Place of birth', 'owl#inverseOf', 'People born here'), ('Jessica Simpson', 'music.artist.genre', 'Synthpop'), ('Kid Cudi', 'broadcast.artist.content', '1Club.FM: Power'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_729v'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jeremy Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0101fs_z'), ('Justin bieber', 'common.image.size', 'm.01x59ss'), ('Terence Dudley', 'music.artist.genre', 'Pop music'), ('m.0vb6hhj', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Ice Cube', 'broadcast.artist.content', 'JellyRadio.com'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3cx'), ('Ja Rule', 'broadcast.artist.content', '1Club.FM: Power'), ('Janet Jackson', 'music.artist.genre', 'Rock music'), ('m.0yrkgd6', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Beauty and a Beat', 'common.topic.notable_for', 'g.126stnfcp'), ('Carrie Underwood', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Rodney Jerkins', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft2j'), ('radioIO Todays POP', 'common.topic.image', 'radioio2.gif'), ('Adam Messinger', 'people.person.gender', 'Male'), ('m.0_vmmj6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('RedOne', 'people.person.profession', 'Record producer'), ('Ashley Tisdale', 'people.person.nationality', 'United States of America'), ('Janet Jackson', 'music.artist.genre', 'Dance-pop'), ('Kylie Minogue', 'people.person.profession', 'Artist'), ('Never Say Never', 'music.recording.song', 'Never Say Never'), ('Ashlee Simpson', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'common.topic.webpage', 'm.0cq990t'), ('m.0z8755b', 'award.award_honor.honored_for', 'Under the Mistletoe'), ('Shaffer Smith', 'broadcast.artist.content', 'FLOW 103'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0n4rmg7'), ('Young Artists for Haiti', 'music.musical_group.member', 'm.011m26pv'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sara Bareilles'), ('Whitney Houston', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sisqó'), ('Hikaru Utada', 'people.person.languages', 'English Language'), ('Ronald Isley', 'people.person.gender', 'Male'), ('m.0z898w6', 'award.award_honor.award_winner', 'Justin Bieber'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO Todays POP'), ('Ricky Nelson', 'people.person.profession', 'Singer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Tom Cloud'), ('Teen Choice Award for Choice Red Carpet Fashion Icon Male', 'award.award_category.winners', 'm.0z87d3n'), ('Brit Award for International Breakthrough Act', 'award.award_category.winners', 'm.0y803nt'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Demi Lovato'), ('Book', 'freebase.type_profile.strict_included_types', 'Written Work'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Capone'), ('Musician', 'people.profession.specializations', 'Singer-songwriter'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Thought Of You', 'common.topic.notable_for', 'g.125525fwv'), ('Right Here', 'music.recording.artist', 'Drake'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0n1ykxp'), ('Coldplay', 'music.artist.genre', 'Indie rock'), ('181-beat', 'broadcast.content.genre', 'Hip hop music'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Dance music'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Estelle', 'music.artist.genre', 'Pop music'), ('m.0101ftq9', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Pras', 'common.topic.notable_types', 'Musical Artist'), ('1Club.FM: 80s (Pop)', 'common.topic.image', '1clubfm.jpg'), ('Justin Bieber', 'celebrities.celebrity.legal_entanglements', 'm.0_cyzs_'), ('Fabian', 'people.person.profession', 'Actor'), ('m.0101fv8j', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('Die in Your Arms', 'music.recording.releases', 'Believe'), ('My World', 'award.award_winning_work.awards_won', 'm.0yr9c1k'), ('Alanis Morissette', 'people.person.profession', 'Film Producer'), ('Canadian', 'common.topic.notable_for', 'g.1255pjt30'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Alicia Keys'), ('m.0j_tkcq', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Never Say Never', 'common.topic.notable_types', 'Musical Album'), ('m.0gc_9w6', 'common.webpage.resource', 'Justin Bieber Videos'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z7n7h'), ('Anastacia', 'common.topic.notable_types', 'Musical Artist'), ('m.0gxnny8', 'people.place_lived.person', 'Justin Bieber'), ('P!nk', 'music.artist.genre', 'Hip hop music'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Live My Life', 'music.single.versions', 'Live My Life'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Pray'), ('P!nk', 'broadcast.artist.content', 'Sunshine Radio'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m5x'), ('Hold Tight', 'music.composition.composer', 'James Giannos'), ('Britney Spears', 'music.artist.genre', 'Electronic music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z898w6'), ('Smoothbeats', 'broadcast.content.genre', 'Contemporary R&B'), ('m.0wjhc6c', 'award.award_honor.award', 'Teen Choice Award for Choice Twitter Personality'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Frank Sinatra', 'people.person.profession', 'Actor'), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Estelle', 'people.person.gender', 'Female'), ('Parents', 'type.property.schema', 'Person'), ('Justin Bieber: Never Say Never', 'film.film.sequel', \"Justin Bieber's Believe\"), ('Never Let You Go', 'music.composition.recordings', 'Never Let You Go'), ('#Thatpower', 'music.recording.artist', 'Justin Bieber'), ('WildFMRadio.com', 'broadcast.content.genre', 'Rap music'), ('Carrie Underwood', 'broadcast.artist.content', 'radioIO Todays POP'), ('Van Halen', 'broadcast.artist.content', 'Sunshine Radio'), ('Lady Gaga', 'broadcast.artist.content', 'Hot Wired Radio'), ('Baby', 'music.composition.recordings', 'Baby'), ('Mary J. Blige', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Pattie Mallette', 'book.author.works_written', \"Nowhere but Up: The Story of Justin Bieber's Mom\"), ('m.0_x4zg3', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gc_9w6', 'common.webpage.topic', 'Justin Bieber'), ('Pray', 'music.single.versions', 'Baby (acoustic)'), ('Ice Cube', 'people.person.languages', 'English Language'), ('Runaway Love (remix)', 'music.recording.tracks', 'Runaway Love (remix)'), ('Kylie Minogue', 'broadcast.artist.content', 'Sunshine Radio'), ('1.FM Top 40', 'broadcast.content.artist', 'Wayne Wonder'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('As Long As You Love Me (Audien Luvstep mix)', 'common.topic.notable_types', 'Musical Recording'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Massimo Boldi'), ('Soulja Boy', 'people.person.place_of_birth', 'Chicago'), ('Ciara', 'broadcast.artist.content', '1Club.FM: Power'), ('Taylor Swift', 'music.artist.genre', 'Pop music'), ('PowerHitz', 'broadcast.content.artist', 'Sean Paul'), ('PowerHitz', 'broadcast.content.artist', 'Fabolous'), ('Lil Jon', 'people.person.profession', 'Actor'), ('All That Matters', 'music.album.releases', 'All That Matters'), ('Pray', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'Somebody to Love'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Paramore'), ('Duffy', 'people.person.profession', 'Singer-songwriter'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO RNB Mix'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Usher'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Usher'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Avi Youabian'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_vkd'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0z340zt'), ('Dance music', 'broadcast.genre.content', 'Hot 97.7'), ('Musician', 'common.topic.notable_types', 'Profession'), ('Yves Bole', 'base.musicpf.home_page.artist', 'Yves Bole'), ('Synthpop', 'common.topic.notable_types', 'Musical genre'), ('1Club.FM: V101', 'broadcast.content.artist', 'Usher'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Children'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h521z'), ('Contemporary R&B', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('Adrienne Bailon', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftks'), ('Taylor Swift', 'music.artist.genre', 'Rock music'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Indie rock', 'common.topic.subjects', 'Indie rock'), ('Kelis', 'people.person.profession', 'Singer-songwriter'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Never Say Never (acoustic)', 'common.topic.notable_types', 'Musical Recording'), ('Nelly Furtado', 'people.person.profession', 'Actor'), ('My World', 'music.album.genre', 'Pop music'), ('Justin Bieber: Never Say Never', 'common.topic.notable_types', 'Award-Winning Work'), ('All Around the World', 'music.album.primary_release', 'All Around the World'), ('Boyfriend (acoustic version)', 'music.recording.song', 'Boyfriend'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Will i Am', 'award.award_nominee.award_nominations', 'm.0y84ynj'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m38'), ('m.0_w1gn3', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0njvvj_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Twista'), ('Christina Milian', 'music.artist.label', 'Island Records'), ('Gwen Stefani', 'people.person.gender', 'Female'), ('2010 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw1tn'), ('P!nk', 'broadcast.artist.content', 'Hot Wired Radio'), ('Jordin Sparks', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('P!nk', 'people.person.nationality', 'United States of America'), ('All Bad', 'music.album.release_type', 'Single'), ('Sean Kingston', 'common.topic.notable_types', 'Musical Artist'), ('K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Hot Wired Radio', 'broadcast.content.artist', 'KT Tunstall'), ('Eminem', 'broadcast.artist.content', '1Club.FM: Channel One'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft0d'), ('Gavin DeGraw', 'people.person.profession', 'Singer'), ('Jessie J', 'common.topic.notable_types', 'Musical Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Brandy Norwood'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0v90skf'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Daniel Bedingfield', 'music.artist.genre', 'Dance-pop'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Teyana'), ('Foreign Remix', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.episode', 'Finalists Chosen #1'), ('Believe Tour', 'common.topic.notable_for', 'g.1jffqwjl0'), ('The Roots', 'broadcast.artist.content', 'Smoothbeats'), ('Justin Bieber: Rise to Fame', 'film.film.personal_appearances', 'm.0nh04xx'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Christina Milian'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('m.0yrjynf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('The Isley Brothers', 'music.artist.genre', 'Quiet Storm'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.artist', 'Justin Bieber'), ('MTV Movie Award for Best Jaw Dropping Moment', 'award.award_category.winners', 'm.0pc670l'), ('Tricky Stewart', 'people.person.nationality', 'United States of America'), ('1.FM Top 40', 'broadcast.content.artist', 'Evanescence'), ('Everlast', 'music.artist.label', 'The Island Def Jam Music Group'), ('Change Me', 'music.album.artist', 'Justin Bieber'), ('Pop music', 'common.topic.notable_types', 'Musical genre'), ('Ice Cube', 'broadcast.artist.content', 'Smoothbeats'), ('As Long as You Love Me', 'music.composition.form', 'Song'), ('The Black Eyed Peas', 'broadcast.artist.content', 'FLOW 103'), ('Lil Wayne', 'music.artist.genre', 'Rhythm and blues'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Never Say Never', 'music.album.primary_release', 'Never Say Never'), ('1.FM Top 40', 'broadcast.content.artist', 'Goo Goo Dolls'), ('Singer-songwriter', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('End Date', 'rdf-schema#range', 'Date/Time'), ('Confident', 'music.composition.composer', 'Verse Simmonds'), ('Beautiful', 'music.single.versions', 'Beautiful'), ('Shaggy', 'music.artist.genre', 'World music'), ('Clay Aiken', 'music.artist.genre', 'Pop music'), ('Kelly Clarkson', 'broadcast.artist.content', '.977 The Hits Channel'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'music.recording.artist', 'Carly Rae Jepsen'), (\"1Club.FM: Jammin' Oldies\", 'common.topic.image', '1clubfm.jpg'), ('Hot Wired Radio', 'common.topic.image', 'c.jpg'), ('m.0vp755b', 'music.recording_contribution.contributor', 'Nicki Minaj'), ('My Worlds', 'music.album.artist', 'Justin Bieber'), ('Terius Nash', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Shaffer Smith', 'broadcast.artist.content', '.977 The Hits Channel'), ('J. Holiday', 'people.person.gender', 'Male'), ('Boyfriend', 'music.album.primary_release', 'Boyfriend'), ('J. Holiday', 'people.person.nationality', 'United States of America'), ('#Thatpower', 'music.recording.tracks', '#Thatpower'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ja Rule'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Anastacia', 'music.artist.genre', 'Pop music'), ('Bigger', 'music.composition.composer', 'Kevin Risto'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhhqv'), ('Rihanna', 'freebase.valuenotation.has_no_value', 'Children'), ('Sean Kingston', 'people.person.nationality', 'United States of America'), ('m.0h17ns7', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('m.0ghz3d6', 'tv.tv_guest_role.episodes_appeared_in', 'Cheryl Cole and Alicia Keys @ R1BW'), ('Anastacia', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('JellyRadio.com', 'broadcast.content.artist', 'Ice Cube'), ('Hold Tight', 'music.album.primary_release', 'Hold Tight'), ('m.0t4s_bn', 'award.award_honor.ceremony', 'Juno Awards of 2013'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Big R Radio - The Hawk', 'common.topic.notable_for', 'g.125500lwf'), ('m.0_svs8m', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Never Let You Go', 'common.topic.notable_types', 'Canonical Version'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.composition.recordings', 'Baby'), ('Sheryl Crow', 'music.artist.genre', 'Rock music'), ('Jessica Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Teen idol', 'base.icons.icon_genre.icons', 'Tommy Sands'), ('Contemporary R&B', 'broadcast.genre.content', '181-party'), ('m.0gz0fr6', 'film.personal_film_appearance.person', 'Pattie Mallette'), ('m.0v90skf', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('m.09xh0kp', 'common.webpage.topic', 'Record producer'), ('Janet Jackson', 'people.person.profession', 'Actor'), ('Jennifer Lopez', 'people.person.profession', 'Film Producer'), ('Janet Jackson', 'broadcast.artist.content', 'Sunshine Radio'), ('Tommy Sands', 'people.person.profession', 'Actor'), ('m.0y84ynj', 'award.award_nomination.nominated_for', '#thatPower'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chris Brown', 'music.artist.genre', 'Dance-pop'), ('PowerHitz', 'broadcast.content.artist', 'Ciara'), ('181-beat', 'broadcast.content.artist', 'Sean Combs'), ('Hold Tight', 'music.album.artist', 'Justin Bieber'), ('Jay-Z', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Lupe Fiasco', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Elvis Presley', 'people.person.profession', 'Actor'), ('Jordin Sparks', 'people.person.profession', 'Actor'), ('Yves Bole', 'base.fashionmodels.fashion_model.eye_color', 'Brown'), ('Tyga', 'people.person.profession', 'Actor'), ('m.0bnstws', 'common.webpage.in_index', 'Blissful Master Index'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelly Clarkson', 'people.person.profession', 'Singer-songwriter'), ('Big Sean', 'music.artist.genre', 'Hip hop music'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Music executive', 'people.profession.specializations', 'Record producer'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Sia Furler', 'music.artist.genre', 'Indie rock'), ('m.0sgkyfg', 'award.award_honor.award_winner', 'Justin Bieber'), ('Next to You', 'music.album.compositions', 'Next to You'), ('Usher', 'broadcast.artist.content', '181-beat'), ('Nicki Minaj', 'music.featured_artist.albums', 'Beauty and a Beat (Remixes)'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Chingy', 'broadcast.artist.content', 'PowerHitz'), ('Bigger', 'music.composition.composer', 'Dapo Torimiro'), (\"Justin Bieber's Believe\", 'film.film.genre', 'Music'), ('Britney Spears', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Lady Antebellum', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), (\"Justin Bieber's Believe\", 'film.film.language', 'English Language'), ('Avril Lavigne', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('.977 The Hits Channel', 'broadcast.content.artist', \"Destiny's Child\"), ('Paul Anka', 'people.person.nationality', 'Canada'), ('Hikaru Utada', 'music.artist.genre', 'Pop music'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All That Matters', 'music.recording.artist', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Frankie J'), ('Next to You', 'music.recording.canonical_version', 'Next to You'), ('Jordan Francis', 'people.person.profession', 'Dancer'), ('Lupe Fiasco', 'people.person.gender', 'Male'), ('Bryan Adams', 'music.artist.genre', 'Rock music'), ('Paul Anka', 'base.icons.icon.icon_genre', 'Teen idol'), ('Jaden Smith', 'music.featured_artist.albums', 'Never Say Never'), ('Beautiful', 'music.recording.canonical_version', 'Beautiful'), ('Van Halen', 'music.artist.genre', 'Rock music'), ('JoJo', 'music.artist.genre', 'Pop music'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('St. Michael Catholic Secondary School', 'education.educational_institution.students_graduates', 'm.0w5l5h1'), ('Bryan Adams', 'people.person.nationality', 'Canada'), ('Janet Jackson', 'people.person.profession', 'Singer-songwriter'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010n0hfx'), ('All Around The World (featuring Ludacris)', 'common.topic.notable_types', 'Musical Recording'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('Kelly Clarkson', 'people.person.profession', 'Record producer'), ('Person', 'type.type.properties', 'Date of birth'), ('Will Smith', 'people.person.children', 'Jaden Smith'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Profession'), ('NME Award for Worst Dressed', 'award.award_category.winners', 'm.0z8s562'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)'), ('HitzRadio.com', 'broadcast.content.artist', '50 Cent'), ('Colbie Caillat', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'common.topic.notable_types', 'Musical Recording'), ('NME Awards 2012', 'award.award_ceremony.awards_presented', 'm.0z83x9l'), ('Hot Wired Radio', 'broadcast.content.genre', 'Pop music'), ('Live My Life', 'common.topic.notable_for', 'g.11b75k11dz'), ('School Boy Records', 'common.topic.notable_for', 'g.1258hxrpm'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('HitzRadio.com', 'broadcast.content.artist', 'Shaggy'), ('Weblink', 'rdf-schema#domain', 'Topic'), ('Justin Bieber', 'base.popstra.celebrity.infidelity_victim', 'm.0gxnp2l'), ('Stratford', 'location.location.containedby', 'Ontario'), ('Avril Lavigne', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Pop music'), ('Mary J. Blige', 'people.person.profession', 'Model'), ('Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Stephen Melton', 'people.person.profession', 'Musician'), ('m.0ws63cr', 'award.award_nomination.award_nominee', 'The Island Def Jam Music Group'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Gas Pedal', 'music.recording.song', 'Gas Pedal'), ('Recovery', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('m.0102z0vx', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Kelis', 'music.artist.genre', 'Hip hop music'), ('Lil Wayne', 'people.person.profession', 'Record producer'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('DMX', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Power'), ('FLOW 103', 'broadcast.content.artist', 'Tupac Shakur'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lil Wayne'), ('Demi Lovato', 'people.person.profession', 'Actor'), ('Justin Timberlake', 'common.topic.notable_types', 'Musical Artist'), ('1.FM Top 40', 'broadcast.content.artist', 'Simple Plan'), ('Europop', 'common.topic.notable_types', 'Musical genre'), ('Verse Simmonds', 'common.topic.notable_types', 'Musical Artist'), ('Never Say Never', 'music.recording.artist', 'Justin Bieber'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Mariah Carey', 'people.person.nationality', 'United States of America'), ('Right Here (featuring Drake)', 'music.recording.releases', 'Believe'), ('m.07053l5', 'common.webpage.resource', 'm.0bm1zq9'), ('Bad Day', 'music.composition.composer', 'Ronald Isley'), ('David Cassidy', 'people.person.profession', 'Actor'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Rob Thomas', 'freebase.valuenotation.has_value', 'Parents'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Kelly Clarkson'), ('m.0101ft1r', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Scooter Braun'), ('JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw59_'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkc0l'), ('Tommy Sands', 'people.person.gender', 'Male'), ('m.0n58kgb', 'award.award_nomination.ceremony', 'American Music Awards of 2012'), ('Duffy', 'music.artist.genre', 'Pop music'), ('Taylor Swift', 'broadcast.artist.content', '1.FM Top 40'), ('Gender', 'rdf-schema#domain', 'Person'), ('Donna Summer', 'people.person.profession', 'Musician'), ('Ludacris', 'music.composer.compositions', 'All Around The World'), ('Dancer', 'people.profession.specialization_of', 'Artist'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010n0htg'), ('Die in Your Arms', 'music.single.versions', 'Die in Your Arms'), ('Miley Cyrus', 'music.artist.genre', 'Pop music'), ('Usher', 'people.person.gender', 'Male'), ('1Club.FM: Power', 'broadcast.content.artist', 'Lil Jon'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Rehab'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Never Say Never', 'music.album.release_type', 'Single'), ('m.0r90dwg', 'common.webpage.category', 'Topic Webpage'), ('C1', 'people.person.nationality', 'United States of America'), ('Canadian', 'people.ethnicity.includes_groups', 'Scottish Canadian'), ('Justin Bieber: Never Say Never', 'film.film.language', 'English Language'), ('Amerie', 'common.topic.notable_types', 'Musical Artist'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Keyshia Cole', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Mann', 'people.person.nationality', 'United States of America'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('K-Ci & JoJo', 'broadcast.artist.content', '1Club.FM: Power'), ('Film Producer', 'common.topic.notable_types', 'Profession'), ('RedOne', 'music.artist.genre', 'Contemporary R&B'), ('Chloe Bridges', 'people.person.profession', 'Actor'), ('Juelz Santana', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Indie rock', 'common.topic.subject_of', 'Pop music'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'The Fray'), ('Annette Funicello', 'people.person.gender', 'Female'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Tupac Shakur'), ('Britney Spears', 'music.artist.genre', 'Dance music'), ('Lil Wayne', 'people.person.languages', 'English Language'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Baby Bash'), ('Teen idol', 'common.topic.webpage', 'm.09y8cm9'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0101fvfh', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Mariah Carey', 'common.topic.notable_types', 'Musical Artist'), ('Rhythm and blues', 'music.genre.subgenre', 'K-pop'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.composer.compositions', 'First Dance'), ('2011 MTV Movie Awards', 'time.event.locations', 'United States of America'), ('Justin Bieber', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Avril Lavigne', 'people.person.nationality', 'Canada'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('m.0sgkw8v', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('Mary J. Blige', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0101ftrz', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Die in Your Arms'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Kerrang! Awards 2012', 'award.award_ceremony.awards_presented', 'm.0y7y53r'), ('Marvin Isley', 'freebase.valuenotation.has_value', 'Parents'), ('FLOW 103', 'broadcast.content.artist', 'Mary J. Blige'), ('Chingy', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'music.composer.compositions', 'Thought Of You'), ('Paul Anka', 'people.person.profession', 'Actor'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('JellyRadio.com', 'broadcast.content.genre', 'Reggae'), ('Ellen DeGeneres', 'people.person.profession', 'Actor'), ('P!nk', 'people.person.gender', 'Female'), ('Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Terius Nash', 'people.person.gender', 'Male'), ('m.0ywsnc9', 'award.award_nomination.ceremony', '5th Annual Shorty Awards'), ('Bryan Adams', 'common.topic.notable_types', 'Musical Artist'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.episode', 'Top 3 Finalists Perform'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0njhyh_', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Juvenile'), ('Tommy Sands', 'people.person.nationality', 'United States of America'), ('Kanye West', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jon M. Chu', 'people.person.nationality', 'United States of America'), ('Katy Perry', 'people.person.nationality', 'United States of America'), ('Lil Wayne', 'music.artist.genre', 'Pop music'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3_7'), ('Chris Brown', 'music.artist.genre', 'Contemporary R&B'), ('A Star Was Born: Justin Bieber', 'film.film.genre', 'Music'), ('Nelly', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Jules & Raoul'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkyfg'), ('Film', 'freebase.type_hints.included_types', 'Topic'), ('Jennifer Lopez', 'music.artist.label', 'The Island Def Jam Music Group'), ('Award-Winning Work', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Instrumental)'), ('m.0y5tj13', 'film.personal_film_appearance.film', '30 Days in May'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('Pras', 'broadcast.artist.content', '.977 The Hits Channel'), ('Weblink', 'type.property.schema', 'Topic'), ('Roller Coaster', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Bad Day', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z340zt'), ('Runaway Love', 'music.recording.song', 'Runaway Love'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ludacris', 'award.award_winner.awards_won', 'm.0n1ykxp'), ('Brandy Norwood', 'people.person.nationality', 'United States of America'), ('Kid Cudi', 'music.artist.genre', 'Rock music'), ('Boyfriend (Neon NiteClub Remix)', 'common.topic.notable_types', 'Musical Recording'), ('Pras', 'people.person.profession', 'Film Producer'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.location', 'Chicago'), ('P!nk', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc3_1'), ('Coldplay', 'broadcast.artist.content', 'radioIO Todays POP'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('m.0hvlt03', 'film.film_film_distributor_relationship.distributor', 'Paramount Pictures'), ('Teen pop', 'common.topic.notable_for', 'g.1259fbz0c'), ('Fabolous', 'people.person.profession', 'Actor'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Outkast'), ('Aaliyah', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Fergie', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.genre', 'Dance music'), ('Rihanna', 'music.artist.genre', 'Synthpop'), ('Ashley Tisdale', 'people.person.profession', 'Film Producer'), ('Michael Jackson', 'music.artist.genre', 'Pop music'), ('Juno Award for Pop Album of the Year', 'award.award_category.winners', 'm.0njvtth'), ('Mary J. Blige', 'music.artist.genre', 'Hip hop music'), ('Kid Cudi', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Vanessa Hudgens', 'people.person.profession', 'Singer'), ('m.0z1scxk', 'award.award_honor.award_winner', 'Justin Bieber'), ('Madonna', 'people.person.nationality', 'United States of America'), ('Juicy J', 'common.topic.notable_types', 'Musical Artist'), ('Ashley Tisdale', 'broadcast.artist.content', 'Hot Wired Radio'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ultrabeat vs. Darren Styles'), ('Under the Mistletoe', 'common.topic.article', 'm.0h7l5vh'), ('Beautiful', 'music.recording.tracks', 'g.11b81rhqyh'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Madonna'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Shaffer Smith'), ('Hikaru Utada', 'music.artist.genre', 'Rhythm and blues'), ('Mary J. Blige', 'people.person.gender', 'Female'), ('Anastacia', 'people.person.profession', 'Dancer'), ('Runaway Love (remix)', 'music.album.featured_artists', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvmb'), ('m.0dj34q5', 'common.webpage.resource', 'Justin Bieber Fan Club'), ('Katy Perry: Part of Me', 'film.film.rating', 'PG (USA)'), ('justinbieber', 'internet.blog.language', 'English Language'), ('Zac Brown Band', 'broadcast.artist.content', 'Hot Wired Radio'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0jvgmxc'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Die in Your Arms', 'music.composition.composer', 'Rodney Jerkins'), ('HitzRadio.com', 'broadcast.content.artist', 'New Kids on the Block'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('Tampa', 'base.biblioness.bibs_location.country', 'United States of America'), ('Chingy', 'music.artist.genre', 'Hip hop music'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.featured_artists', 'Big Sean'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Paramore'), ('Pray', 'music.composition.language', 'English Language'), ('Blackstreet', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108j492'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Lolly', 'music.recording.artist', 'Maejor'), ('m.0njw4z2', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('PowerHitz', 'broadcast.content.genre', 'Hip hop music'), ('My World', 'music.album.primary_release', 'My World'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fs_z'), ('Whitney Houston', 'people.person.profession', 'Singer'), ('Nelly', 'common.topic.notable_types', 'Musical Artist'), ('Britney Spears', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'people.person.education', 'm.0yrlqp1'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Snoop Dogg'), ('Stuart Ford', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Never Say Never'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Madonna'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Dub Mix)'), ('Boyfriend', 'common.topic.notable_types', 'Musical Recording'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k1h'), ('Baby', 'music.composition.composer', 'Justin Bieber'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Profession'), ('iJustine', 'influence.influence_node.influenced', 'Yves Bole'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Big K.R.I.T.'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Brutha'), ('1Club.FM: V101', 'broadcast.content.artist', 'Mary J. Blige'), ('All That Matters', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Max Martin', 'music.artist.genre', 'Contemporary R&B'), ('Ray J', 'freebase.valuenotation.has_no_value', 'Children'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Chris Brown'), ('Kelis', 'common.topic.notable_types', 'Musical Artist'), ('Whitney Houston', 'music.artist.genre', 'Contemporary R&B'), ('m.09y8dts', 'common.webpage.topic', 'Teen idol'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft5f'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Never Say Never', 'common.topic.notable_for', 'g.12555vkyk'), ('Beyoncé Knowles', 'music.artist.genre', 'Dance music'), ('Never Say Never', 'music.composition.composer', 'Justin Bieber'), ('Jaden Smith', 'people.person.profession', 'Singer-songwriter'), ('Jordan Pruitt', 'common.topic.notable_types', 'Musical Artist'), ('Under the Mistletoe', 'music.album.genre', 'Dance-pop'), ('Ciara', 'music.artist.genre', 'Dance music'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All Around the World', 'music.album.artist', 'Justin Bieber'), ('Jane Lipsitz', 'people.person.gender', 'Female'), ('Mann', 'music.artist.label', 'The Island Def Jam Music Group'), ('Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bigger', 'freebase.valuenotation.is_reviewed', 'Lyricist'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Lil Jon', 'people.person.gender', 'Male'), ('m.0j8z6tl', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('m.0njw444', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0101fv4x', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Die in Your Arms', 'music.composition.recordings', 'Die in Your Arms'), ('Musician', 'people.profession.specializations', 'Chansonnier'), ('Adrienne Bailon', 'music.artist.genre', 'Dance music'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('World music', 'music.genre.subgenre', 'Pop music'), ('m.010lkp2z', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('m.09wnjlc', 'common.webpage.topic', 'Teen idol'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Singer-songwriter', 'people.profession.specializations', 'Chansonnier'), ('m.09wf9d1', 'common.webpage.topic', 'Teen idol'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.song', 'As Long as You Love Me'), ('Kevin Risto', 'people.person.nationality', 'Canada'), ('Jazmyn Bieber', 'people.person.place_of_birth', 'Canada'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3dn'), ('Bad Day', 'common.topic.notable_types', 'Composition'), ('HitzRadio.com', 'broadcast.content.artist', 'Big Pun'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('GotRadio - RnB Classics', 'broadcast.content.genre', 'Top 40'), ('Rihanna', 'broadcast.artist.content', '1.FM Top 40'), ('All Around The World', 'music.composition.recordings', 'All Around the World (acoustic version)'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten radio)'), ('Amerie', 'people.person.profession', 'Actor'), ('JellyRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0vpggkk', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Boyfriend', 'music.recording.artist', 'Justin Bieber'), ('Athan Grace', 'people.person.profession', 'Actor'), ('m.0n1ykxp', 'award.award_honor.ceremony', '2010 MTV Video Music Awards'), ('Stuart Ford', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjy9'), ('Pop music', 'common.topic.subjects', 'Indie rock'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Album'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('JellyRadio.com', 'broadcast.content.genre', 'Dance music'), ('m.0gxnp26', 'base.popstra.breakup.participant', 'Justin Bieber'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Canada', 'common.topic.notable_types', 'Country'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Avril Lavigne', 'people.person.gender', 'Female'), ('Lady Gaga', 'music.artist.genre', 'Rock music'), ('m.0_vw6nn', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Jason Mraz'), ('Thought Of You', 'music.recording.song', 'Thought Of You'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw1tn'), ('Sia Furler', 'people.person.gender', 'Female'), ('Kylie Minogue', 'people.person.profession', 'Singer'), (\"Destiny's Child\", 'common.topic.notable_types', 'Musical Artist'), ('Avril Lavigne', 'common.topic.notable_types', 'Musical Artist'), ('Rihanna', 'broadcast.artist.content', '181-beat'), ('Beauty and a Beat (Remixes)', 'music.album.releases', 'Beauty and a Beat (Remixes)'), ('Christina Milian', 'music.artist.label', 'The Island Def Jam Music Group'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Nasri', 'music.artist.genre', 'Rhythm and blues'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kelis', 'music.artist.genre', 'Synthpop'), ('1Club.FM: Power', 'broadcast.content.artist', 'Ciara'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Kate Nash'), ('Stratford', 'location.location.people_born_here', 'Pattie Mallette'), ('Sean Kingston', 'music.composer.compositions', 'Eenie Meenie'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('The Notorious B.I.G.', 'people.person.languages', 'English Language'), ('Justin Bieber', 'music.artist.album', 'U Smile'), ('Right Here', 'music.composition.composer', 'Justin Bieber'), ('Usher', 'broadcast.artist.content', '1Club.FM: V101'), ('City/Town/Village', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'base.schemastaging.person_extra.net_worth', 'm.0j8mhs_'), ('m.0y5tj13', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Singer-songwriter'), ('Profession', 'rdf-schema#range', 'Profession'), ('Shaffer Smith', 'music.artist.genre', 'Pop music'), ('Alanis Morissette', 'people.person.nationality', 'United States of America'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('Jeremy Bieber', 'common.topic.notable_types', 'Person or entity appearing in film'), ('Paul Anka', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kylie Minogue', 'people.person.profession', 'Musician'), ('Gwen Stefani', 'music.artist.genre', 'Reggae'), ('Max Martin', 'people.person.profession', 'Musician'), ('Justin Bieber', 'music.artist.album', 'PYD'), ('Michael Jackson', 'broadcast.artist.content', 'Classic Soul Network'), ('#thatPOWER', 'music.recording.artist', 'Will i Am'), ('Duffy', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Britney Spears', 'people.person.profession', 'Film Producer'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Robby Stewart'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.genre', 'Documentary film'), ('Caitlin Beadles', 'people.person.gender', 'Female'), ('Person', 'type.type.properties', 'Place of birth'), ('Christina Milian', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Zac Efron', 'people.person.nationality', 'United States of America'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Santana'), ('Eenie Meenie', 'common.topic.notable_types', 'Composition'), ('Somebody to Love (remix)', 'music.album.album_content_type', 'Remix album'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Twista', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsc'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Ronski Speed'), ('Hot Wired Radio', 'broadcast.content.artist', 'Fall Out Boy'), ('JellyRadio.com', 'broadcast.content.artist', 'Snoop Dogg'), ('Singer-songwriter', 'people.profession.specialization_of', 'Musician'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 64kbps Stream'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'music.composition.recordings', 'Baby'), ('Rihanna', 'broadcast.artist.content', 'Sunshine Radio'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cielo Drive'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Parents'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft35'), ('Beautiful and the Beat', 'music.recording.artist', 'Justin Bieber'), ('Aaliyah', 'music.artist.genre', 'Rhythm and blues'), ('Will i Am', 'people.person.profession', 'Record producer'), ('Justin Bieber', 'common.topic.webpage', 'm.0dj34q5'), ('m.0njwqrb', 'freebase.valuenotation.has_no_value', 'Winning work'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('The Black Eyed Peas', 'music.artist.genre', 'Trance music'), ('m.0115qhzk', 'award.award_honor.award', 'MTV Europe Music Voices Award'), ('m.0z898w6', 'award.award_honor.honored_for', 'CSI: Crime Scene Investigation'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0vpggkk', 'music.recording_contribution.album', 'We Are the World 25 for Haiti'), ('Avril Lavigne', 'people.person.profession', 'Actor'), ('Frank Ocean', 'music.artist.genre', 'Contemporary R&B'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Victoria Justice'), ('Runaway Love (remix)', 'music.album.contributor', 'm.0vp7m_x'), ('Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'music.single.versions', 'Baby'), ('Trey Songz', 'people.person.gender', 'Male'), ('Where Are Ü Now', 'music.recording.tracks', 'Where Are Ü Now'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Usher', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3cx'), ('m.0115qhzk', 'award.award_honor.award_winner', 'Justin Bieber'), ('Location', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhvsq'), ('Whitney Houston', 'people.person.profession', 'Musician'), ('Miley Cyrus', 'music.artist.genre', 'Teen pop'), ('Yves Bole', 'influence.influence_node.influenced', 'Jessie J'), ('Die in Your Arms', 'music.composition.composer', 'Dennis \\\\\"Aganee\\\\\" Jenkins'), ('Pattie Mallette', 'people.person.profession', 'Film Producer'), ('m.0102z0vx', 'award.award_honor.award_winner', 'Justin Bieber'), ('LeAnn Rimes', 'people.person.gender', 'Female'), ('Gas Pedal', 'music.recording.artist', 'Sage The Gemini'), ('Rob Thomas', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Gavin DeGraw', 'people.person.profession', 'Actor'), ('Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Place of birth'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Britney Spears'), ('#thatPOWER', 'music.album.contributor', 'm.0vpgdpb'), ('Jessica Simpson', 'people.person.profession', 'Singer'), ('Lady Gaga', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juicy J', 'people.person.nationality', 'United States of America'), ('Big R Radio - The Hawk', 'common.topic.notable_types', 'Broadcast Content'), ('Trick Daddy', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'people.person.nationality', 'United States of America'), ('m.0gd28hv', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Sheryl Crow', 'people.person.profession', 'Artist'), (\"Destiny's Child\", 'broadcast.artist.content', '1Club.FM: Mix 106'), ('A Star Was Born: Justin Bieber', 'film.film.genre', 'Documentary film'), ('Ricky Nelson', 'common.topic.notable_types', 'Musical Artist'), ('Ray J', 'people.person.profession', 'Record producer'), ('Italian Language', 'language.human_language.main_country', 'Italy'), ('Baby', 'music.album.artist', 'Justin Bieber'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Profession'), ('American Music Award for New Artist of the Year', 'award.award_category.winners', 'm.0zbf_4g'), ('Enrique Iglesias', 'people.person.languages', 'Spanish Language'), ('Never Say Never (acoustic)', 'music.recording.featured_artists', 'Jaden Smith'), ('Geri Halliwell', 'people.person.profession', 'Singer'), ('Justin Bieber: Just Getting Started', 'book.translated_work.translations', 'Justin Bieber : bara början'), ('HitzRadio.com', 'broadcast.content.artist', 'Sisqó'), ('181-beat', 'broadcast.content.genre', 'Pop music'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'C1'), ('Musical Artist', 'common.topic.article', 'm.01z0mrz'), ('Gas Pedal', 'music.single.versions', 'Gas Pedal'), ('m.0v_714c', 'film.personal_film_appearance.person', 'Justin Bieber'), ('BeirutNights.com Radio', 'common.topic.article', 'm.03hrx6z'), ('m.0t4syfh', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('United States of America', 'base.biblioness.bibs_topic.subsumes', 'United States of America'), ('Nasri', 'people.person.profession', 'Record producer'), ('Donna Summer', 'music.artist.genre', 'Rock music'), ('Baby (acoustic)', 'music.recording.artist', 'Justin Bieber'), ('Somebody to Love', 'common.topic.notable_for', 'g.1yl5kzcrs'), ('Avril Lavigne', 'people.person.languages', 'English Language'), ('m.0d33hsc', 'celebrities.friendship.friend', 'Justin Bieber'), ('Michael Jackson', 'music.artist.genre', 'Rhythm and blues'), ('Kelis', 'music.artist.genre', 'Rhythm and blues'), ('Alanis Morissette', 'broadcast.artist.content', '1.FM Top 40'), ('#thatPOWER', 'music.recording.tracks', '#thatPower'), ('Kanye West', 'music.artist.album', 'Runaway Love (remix)'), ('Hot Wired Radio', 'broadcast.content.genre', 'Urban contemporary'), ('Mariah Carey', 'people.person.profession', 'Singer'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Montell Jordan', 'people.person.gender', 'Male'), ('One Less Lonely Girl', 'common.topic.notable_types', 'Musical Album'), ('Usher', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0gbm3d_', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Never Say Never', 'music.composition.composer', 'Adam Messinger'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'MC Ren'), ('m.0tkc3tj', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0101ft3d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Italian Language', 'language.human_language.countries_spoken_in', 'Italy'), ('Die in Your Arms', 'music.composition.composer', 'Travis Sayles'), ('Aaliyah', 'common.topic.notable_types', 'Musical Artist'), ('1.FM Top 40', 'broadcast.content.artist', 'Uncle Kracker'), ('Baby', 'music.composition.composer', 'Tricky Stewart'), ('Eenie Meenie', 'music.composition.composer', 'Sean Kingston'), ('My World 2.0', 'music.album.album_content_type', 'Studio album'), ('Sheryl Crow', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'music.artist.label', 'RBMG Records'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Santana', 'broadcast.artist.content', 'radioIO Classic RNB'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Adrienne Bailon', 'music.artist.genre', 'Hip hop music'), ('Rihanna', 'people.person.gender', 'Female'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Justin Bieber'), ('Willa Ford', 'people.person.profession', 'Record producer'), ('Red Hot Chili Peppers', 'broadcast.artist.content', 'Hot Wired Radio'), ('L.A. Reid', 'people.person.profession', 'Music executive'), ('Timbaland', 'people.person.profession', 'Musician'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvkw'), ('Redfoo', 'music.artist.genre', 'Hip hop music'), ('Shaggy', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Pras', 'music.artist.genre', 'Hip hop music'), ('Ice Cube', 'people.person.gender', 'Male'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never (acoustic)'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Drake', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Believe', 'music.album.releases', 'Believe'), ('The Island Def Jam Music Group', 'common.topic.article', 'm.04fc6k'), ('Adam Messinger', 'people.person.profession', 'Record producer'), ('Daniel Bedingfield', 'music.artist.genre', 'Pop music'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.episode', 'The Father-In-Law'), ('Beauty and a Beat (Bisbetic Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Stratford', 'location.location.people_born_here', 'Jeremy Bieber'), ('Athan Grace', 'people.person.nationality', 'United States of America'), ('Lolly', 'music.composition.composer', 'Justin Bieber'), ('m.0z340zt', 'award.award_honor.award', 'MTV Europe Music Award for Best World Stage Performance'), ('Timbaland', 'music.artist.genre', 'Rock music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Justin Bieber'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'music.artist.genre', 'Rhythm and blues'), ('Spouse', 'owl#inverseOf', 'Spouse (or domestic partner)'), ('My Worlds', 'music.album.releases', 'My Worlds'), ('Kuk Harrell', 'music.artist.genre', 'Hip hop music'), ('The Notorious B.I.G.', 'people.person.profession', 'Record producer'), ('m.0z898w6', 'freebase.valuenotation.is_reviewed', 'Award category'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Carrie Underwood'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Daniel Bedingfield'), ('J. Holiday', 'people.person.profession', 'Actor'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat'), ('Tyga', 'music.artist.album', 'Wait For a Minute'), ('Katy Perry: Part of Me', 'film.film.executive_produced_by', 'Randy Phillips'), ('Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Thought of You', 'music.recording.canonical_version', 'Thought of You'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Enrique Iglesias', 'music.artist.genre', 'Pop music'), ('m.0129jzth', 'people.sibling_relationship.sibling', 'Yves Bole'), ('Kelly Clarkson', 'broadcast.artist.content', 'HitzRadio.com'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Eenie Meenie', 'music.album.artist', 'Justin Bieber'), ('Britney Spears', 'music.artist.genre', 'Urban contemporary'), ('Timbaland', 'music.artist.genre', 'Dance-pop'), ('Contemporary R&B', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'), ('m.0sgkw8v', 'award.award_honor.award_winner', 'Justin Bieber'), ('Jessica Simpson', 'music.artist.genre', 'Dance-pop'), ('radioIO Todays POP', 'common.topic.image', 'RadioIO.png'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0pcr28j'), ('As Long as You Love Me', 'music.recording.artist', 'Big Sean'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Children'), ('Brandy Norwood', 'music.artist.genre', 'Hip hop music'), ('The Pussycat Dolls', 'broadcast.artist.content', '.977 The Hits Channel'), ('Heartbreaker', 'common.topic.notable_types', 'Canonical Version'), ('Beauty and a Beat (Wideboys Dub)', 'music.recording.song', 'Beauty And A Beat'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Chance the Rapper', 'music.composer.compositions', 'Confident'), ('Carrie Underwood', 'music.artist.genre', 'Pop music'), ('Aaliyah', 'people.person.profession', 'Dancer'), ('My World 2.0', 'common.topic.notable_for', 'g.12559ncd9'), ('.977 The Hits Channel', 'broadcast.content.genre', 'Rock music'), ('Blu Cantrell', 'common.topic.notable_types', 'Musical Artist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The All-American Rejects'), ('My World 2.0', 'music.album.releases', 'My World'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.featured_artists', 'Big Sean'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgk_cw'), ('1.FM Top 40', 'broadcast.content.genre', 'Dance music'), ('Jason Mraz', 'people.person.gender', 'Male'), ('1Club.FM: Power', 'broadcast.content.artist', 'Lady Gaga'), ('m.0zbf_4g', 'award.award_honor.award', 'American Music Award for New Artist of the Year'), ('First Dance', 'common.topic.notable_for', 'g.1z2sqd4jr'), ('Duffy', 'common.topic.notable_types', 'Musical Artist'), ('Chance the Rapper', 'people.person.profession', 'Singer-songwriter'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jennifer Lopez', 'music.artist.genre', 'Dance music'), ('Janet Jackson', 'people.person.nationality', 'United States of America'), ('m.010lkp2z', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Chingy', 'people.person.nationality', 'United States of America'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', \"00's\"), ('Parents', 'type.property.reverse_property', 'Children'), ('The Island Def Jam Music Group', 'music.record_label.artist', '6 Tre G'), ('Nelly Furtado', 'music.artist.genre', 'Dance-pop'), ('Spouse (or domestic partner)', 'type.property.schema', 'Person'), ('Whitney Houston', 'people.person.gender', 'Female'), ('Ray J', 'music.artist.genre', 'Contemporary R&B'), ('m.0gh0fdt', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Justin Bieber', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Michael Jackson', 'people.person.profession', 'Actor'), ('John Mamann', 'people.person.profession', 'Music Producer'), ('m.0pc670l', 'award.award_honor.award_winner', 'Justin Bieber'), ('Sean Kingston', 'broadcast.artist.content', 'Hot Wired Radio'), ('Recovery', 'music.album.primary_release', 'Recovery'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bad Day', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('One Less Lonely Girl', 'music.album.genre', 'Pop music'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Taylor Swift'), ('m.0yrjvlh', 'freebase.valuenotation.has_no_value', 'Winning work'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO Classic RNB'), ('HitzRadio.com', 'broadcast.content.genre', 'Pop music'), ('JoJo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Stratford', 'base.wikipedia_infobox.settlement.area_code', 'Area codes 519 and 226'), ('181-beat', 'broadcast.content.artist', 'Outkast'), ('Ricky Nelson', 'people.person.gender', 'Male'), ('Rhythm and blues', 'music.genre.subgenre', 'Pop music'), ('Bryan Adams', 'people.person.gender', 'Male'), ('Never Let You Go', 'music.album.release_type', 'Single'), ('HitzRadio.com', 'broadcast.content.artist', 'Aaliyah'), ('#Thatpower', 'common.topic.notable_for', 'g.12h32d843'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Power'), ('First Dance', 'common.topic.notable_types', 'Musical Recording'), ('P!nk', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'JellyRadio.com'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Paul'), ('Justin Bieber', 'common.topic.notable_types', 'Musical Artist'), ('Will i Am', 'music.artist.genre', 'Pop music'), ('HitzRadio.com', 'broadcast.content.artist', 'Rihanna'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('2011 MTV Video Music Awards', 'award.award_ceremony.awards_presented', 'm.0n4rmg7'), ('All That Matters', 'music.composition.recordings', 'All That Matters'), ('181-beat', 'broadcast.content.genre', 'Hip hop music'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0t4s_bn'), ('m.0njw257', 'award.award_honor.ceremony', '2010 MTV Europe Music Awards'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me (acoustic version)'), ('Linkin Park', 'music.artist.genre', 'Rock music'), ('Heartbreaker', 'common.topic.notable_for', 'g.1ydnq397t'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Nelly'), ('m.0ndc0sf', 'award.award_honor.award_winner', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'Kanye West'), ('1Club.FM: Power', 'broadcast.content.artist', 'Justin Timberlake'), ('m.0cq990t', 'common.webpage.category', 'Topic Webpage'), ('m.0z87d3n', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Disco'), ('m.0sxhgqj', 'award.award_nomination.nominated_for', 'Believe'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flobots'), ('Heartbreaker', 'common.topic.notable_types', 'Musical Album'), ('Turn to You (Mother’s Day Dedication)', 'music.album.primary_release', 'Turn to You (Mother’s Day Dedication)'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.song', 'As Long as You Love Me'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.episode', 'Make Your Mark: Shake It Up Dance Off 2012'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'New Kids on the Block'), ('Italy', 'common.topic.notable_types', 'Country'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mr. M.'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Will Smith', 'music.artist.genre', 'Hip hop music'), ('Iggy Azalea', 'music.artist.genre', 'Electronic music'), ('Gavin DeGraw', 'people.person.nationality', 'United States of America'), ('Snoop Dogg', 'broadcast.artist.content', '1Club.FM: Power'), ('Ashlee Simpson', 'music.artist.genre', 'Rock music'), ('Chris Brown', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audiobot remix)'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0101ft2j', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Boyfriend', 'music.album.releases', 'Boyfriend (Dada Life remix)'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Christmas in Washington', 'film.film.language', 'English Language'), ('Hot Wired Radio', 'common.topic.notable_for', 'g.125fs0kff'), ('Leonardo DiCaprio', 'people.person.profession', 'Film Producer'), ('Bad 25', 'film.film.personal_appearances', 'm.0nfnx_3'), ('Sean Combs', 'music.artist.genre', 'Rhythm and blues'), ('Justin Bieber', 'music.composer.compositions', 'Baby'), ('Chris Brown', 'broadcast.artist.content', 'radioIO Todays POP'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Kid Cudi', 'music.artist.genre', 'Hip hop music'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0_sxby0'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lesley Roy'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvk9'), ('1Club.FM: V101', 'broadcast.content.artist', 'Shaffer Smith'), ('Baby', 'music.music_video.artist', 'Justin Bieber'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc259'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvfh'), ('Home to Mama', 'music.composition.composer', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Christina Milian'), ('End Date', 'type.property.expected_type', 'Date/Time'), ('Teyana', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0vp7qr5', 'music.recording_contribution.album', 'Never Say Never'), ('Somebody to Love (remix)', 'common.topic.notable_for', 'g.126tjdgsh'), ('Hot Wired Radio', 'broadcast.content.artist', 'Rihanna'), ('Justin Bieber', 'music.artist.label', 'School Boy Records'), ('Anastacia', 'people.person.profession', 'Record producer'), ('All Around The World', 'music.composition.lyricist', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Pearl Jam'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('m.0gxnp26', 'base.popstra.breakup.participant', 'Caitlin Beadles'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Children'), ('JoJo', 'people.person.profession', 'Actor'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Power'), ('Never Say Never', 'music.composition.composer', 'Kuk Harrell'), ('Kylie Minogue', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Alicia Keys', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('m.0z3tqqt', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('All Bad', 'music.composition.composer', 'Andre Harris'), ('Britney Spears', 'influence.influence_node.influenced', 'Selena Gomez'), ('Ja Rule', 'people.person.profession', 'Musician'), ('m.0y5tj13', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Benny Blanco', 'music.artist.genre', 'Dance-pop'), ('Taylor Swift', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('m.0z84kwx', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108gyct'), ('Roller Coaster', 'music.composition.composer', 'Justin Bieber'), ('Under the Mistletoe', 'common.topic.image', 'justin-bieber-under-the-mistletoe-album-cover.jpg'), ('PowerHitz', 'broadcast.content.artist', 'Terius Nash'), ('1Club.FM: Power', 'broadcast.content.artist', 'Mariah Carey'), ('Hold Tight', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jll'), ('Donna Summer', 'broadcast.artist.content', 'SoulfulClassics.com'), ('Fergie', 'broadcast.artist.content', 'radioIO Todays POP'), ('P!nk', 'people.person.profession', 'Actor'), ('Yves Bole', 'people.person.languages', 'English Language'), ('Aaliyah', 'music.group_member.instruments_played', 'Vocals'), ('Hip hop music', 'broadcast.genre.content', 'WildFMRadio.com'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkr34'), ('Savan Kotecha', 'people.person.nationality', 'United States of America'), ('Right Here', 'music.single.versions', 'Right Here (featuring Drake)'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Mis-Teeq'), ('Frank Ocean', 'music.lyricist.lyrics_written', 'Bigger'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftq9'), ('Reed Smoot', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Sir Mix-a-Lot', 'people.person.nationality', 'United States of America'), ('Khalil', 'music.artist.label', 'The Island Def Jam Music Group'), ('Emphatic Radio.com!', 'broadcast.content.artist', \"Plain White T's\"), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), ('Justin Bieber', 'people.person.nationality', 'Canada'), ('Madonna', 'broadcast.artist.content', '1.FM Top 40'), ('Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Date of birth'), ('Live My Life (Jaywalker remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Nick Jonas'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.artist', 'Sean Kingston'), ('m.0gxnp2l', 'base.popstra.infidelity.victim', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Height'), ('HitzRadio.com', 'broadcast.content.artist', 'Green Day'), ('Stephen Melton', 'music.artist.genre', 'Rhythm and blues'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.episode', 'Season 1, episode 1'), ('Toby Gad', 'music.artist.genre', 'Pop music'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Radio Mix)'), ('Kings of Leon', 'influence.influence_node.influenced_by', 'Pearl Jam'), ('Country of nationality', 'type.property.expected_type', 'Country'), ('All That Matters', 'music.album.primary_release', 'All That Matters'), ('Blackstreet', 'music.artist.genre', 'Pop music'), ('Tricky Stewart', 'music.artist.genre', 'Contemporary R&B'), ('Adam Messinger', 'music.composer.compositions', 'Next to You'), ('Akon', 'people.person.profession', 'Musician'), ('m.0sgkrj4', 'award.award_honor.ceremony', \"2013 Kids' Choice Awards\"), ('.977 The Hits Channel', 'broadcast.content.artist', 'My Chemical Romance'), ('Selena Gomez', 'music.artist.genre', 'Dance music'), ('Daniel Bedingfield', 'people.person.profession', 'Singer-songwriter'), ('Nick Jonas', 'music.artist.genre', 'Pop music'), ('Christina Aguilera', 'people.person.profession', 'Actor'), ('Max Martin', 'people.person.profession', 'Record producer'), ('CSI: Crime Scene Investigation', 'award.award_winning_work.awards_won', 'm.0z898w6'), ('Believe', 'music.album.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Fabian'), ('Janet Jackson', 'broadcast.artist.content', 'radioIO Todays POP'), ('New Kids on the Block', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Brandy Norwood', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z0tmyv'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me'), ('Record producer', 'common.topic.notable_types', 'Profession'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhyh_'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.song', 'As Long as You Love Me'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Nelly'), ('Britney Spears', 'people.person.profession', 'Dancer'), ('m.0yrhhqv', 'award.award_honor.award', 'MTV Video Music Brazil Award for Best International Artist'), ('Michael Jackson', 'people.person.languages', 'English Language'), ('Justin Bieber', 'common.topic.webpage', 'm.0d_hbgr'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber', 'common.topic.webpage', 'm.0bnstws'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Brooklyn Bounce'), ('Singer', 'people.profession.specializations', 'Playback Singer'), ('Justin Bieber', 'people.person.profession', 'Singer-songwriter'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Flyleaf'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Somebody to Love (remix)', 'music.album.release_type', 'Single'), ('Anastacia', 'people.person.profession', 'Singer-songwriter'), ('m.0z8755b', 'award.award_honor.award', 'NME Award for Worst Album'), ('Anastacia', 'people.person.profession', 'Singer'), ('m.0bm1zq9', 'common.resource.annotations', 'm.07053l5'), ('Hot Wired Radio', 'broadcast.content.artist', 'Gwen Stefani'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'JoJo'), ('Hikaru Utada', 'people.person.profession', 'Singer'), ('Avril Lavigne', 'people.person.profession', 'Artist'), ('Somebody to Love', 'music.composition.recordings', 'Somebody To Love'), ('1Club.FM: V101', 'broadcast.content.genre', 'Urban contemporary'), ('Hip hop music', 'broadcast.genre.content', 'Smoothbeats'), ('1.FM Top 40', 'broadcast.content.broadcast', '1.FM Top 40 - 32kbps Stream'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Juelz Santana'), ('Chloe Bridges', 'people.person.nationality', 'United States of America'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Pop music'), ('Wait For a Minute', 'music.album.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Justin Bieber'), ('Beauty and a Beat', 'music.album.contributor', 'm.0vp755b'), ('1Club.FM: Power', 'broadcast.content.artist', 'Baby Bash'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Mary J. Blige', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: Power'), ('Soulja Boy', 'people.person.languages', 'English Language'), ('m.0101fvjw', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Montell Jordan', 'freebase.valuenotation.has_value', 'Parents'), ('m.0yrkr34', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber: Never Say Never', 'award.award_nominated_work.award_nominations', 'm.0pc67ly'), ('All That Matters', 'music.recording.song', 'All That Matters'), ('Beyoncé Knowles', 'people.person.nationality', 'United States of America'), ('Demi Lovato', 'people.person.nationality', 'United States of America'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.featured_artist.recordings', 'Wait For a Minute'), ('Madonna', 'people.person.profession', 'Actor'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Janet Jackson', 'people.person.profession', 'Model'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Michael Jackson', 'people.person.profession', 'Film Producer'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Record producer', 'common.topic.subject_of', 'Artfarm'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fabolous', 'freebase.valuenotation.has_value', 'Children'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Kylie Minogue'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jeremy Bieber', 'people.person.gender', 'Male'), ('Beautiful and the Beat', 'music.recording.artist', 'Nicki Minaj'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Love Never Felt So Good', 'common.topic.notable_types', 'Composition'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'André Visior'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc259'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Yves Bole', 'base.musicpf.artist.home_page', 'Yves Bole'), ('Cris Cab', 'music.artist.genre', 'Reggae'), ('Jessica Simpson', 'music.artist.genre', 'Rhythm and blues'), ('Believe', 'music.album.releases', 'Believe'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Miami Beach', 'location.location.containedby', 'United States of America'), ('Timbaland', 'music.artist.genre', 'Hip hop music'), ('Janet Jackson', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('m.0102z0vx', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Justin Bieber', 'music.artist.track_contributions', 'm.0rqp4h0'), ('Estelle', 'music.artist.genre', 'Rhythm and blues'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Sir Mix-a-Lot', 'people.person.languages', 'English Language'), ('Geri Halliwell', 'music.artist.genre', 'Dance music'), ('HitzRadio.com', 'broadcast.content.artist', 'Justin Timberlake'), ('Miley Cyrus', 'people.person.gender', 'Female'), ('Person', 'type.type.properties', 'Height'), ('My World 2.0', 'music.album.primary_release', 'My World 2.0'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('L.A. Reid', 'people.person.profession', 'Film Producer'), ('Ciara', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Electronic dance music', 'music.genre.subgenre', 'Trance music'), ('Juelz Santana', 'people.person.gender', 'Male'), ('m.0_grmxj', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Geri Halliwell', 'music.artist.genre', 'Dance-pop'), ('WildFMRadio.com', 'broadcast.content.artist', 'Mims'), ('Shaffer Smith', 'people.person.profession', 'Musician'), ('Reggae', 'common.topic.notable_types', 'Musical genre'), ('HitzRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0wjhc6c', 'freebase.valuenotation.has_no_value', 'Winning work'), ('1.FM Top 40', 'broadcast.content.artist', 'Staind'), ('One Time', 'music.album.artist', 'Justin Bieber'), ('Hot Wired Radio', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Judy Garland', 'people.person.profession', 'Actor'), ('Blu Cantrell', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('CL', 'freebase.valuenotation.has_value', 'Parents'), ('Driving under the influence', 'celebrities.reason_for_arrest.celebrities_charged_or_arrested', 'm.0_cz5l3'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kelly Clarkson', 'people.person.profession', 'Singer'), ('Pattie Mallette', 'people.person.nationality', 'Canada'), ('Mariah Carey', 'music.artist.genre', 'Contemporary R&B'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ice Cube'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0gwhmhm'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Rossano Prini \\\\\"DJ Ross\\\\\"'), ('Kelis', 'people.person.profession', 'Singer'), ('Ginuwine', 'people.person.nationality', 'United States of America'), ('Confident', 'music.album.releases', 'Confident'), ('m.0z87597', 'award.award_nomination.award', 'NME Award for Worst Album'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrktlv'), ('Donna Summer', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Person', 'type.type.properties', 'Gender'), ('Bryan-Michael Cox', 'music.artist.genre', 'Hip hop music'), ('Change Me', 'common.topic.notable_types', 'Musical Recording'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will i Am', 'music.artist.genre', 'Electronic dance music'), ('m.0z0tgz6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.09jx30f', 'base.sounds.audio_recording_contribution.role', 'Record producer'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jay-Z', 'people.person.languages', 'English Language'), ('Gwen Stefani', 'music.artist.genre', 'Contemporary R&B'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('Record producer', 'people.profession.specialization_of', 'Music Producer'), ('Justin Bieber: Never Say Never', 'film.film.runtime', 'm.0gbm35z'), ('As Long as You Love Me', 'music.recording.releases', 'Believe'), ('KooL CrAzE', 'music.artist.genre', 'Rhythm and blues'), ('Pearl Jam', 'music.artist.origin', 'United States of America'), ('MTV Europe Music Voices Award', 'award.award_category.winners', 'm.0115qhzk'), ('m.0_srv2b', 'award.award_nomination.nominated_for', 'justinbieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Ray J'), ('m.0vp8rhw', 'music.recording_contribution.contributor', 'Nicki Minaj'), ('Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhrwc'), ('#thatPower', 'music.recording.canonical_version', '#thatPOWER'), ('Rita Ora', 'music.artist.genre', 'Dance music'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Klubbdriver'), ('The Isley Brothers', 'broadcast.artist.content', 'GotRadio - RnB Classics'), ('Person or entity appearing in film', 'freebase.type_hints.included_types', 'Topic'), ('Usher', 'music.artist.genre', 'Dance music'), ('Kings of Leon', 'music.artist.label', 'Island Records'), ('1Club.FM: Power', 'broadcast.content.artist', 'Tupac Shakur'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njwqrb'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Die in Your Arms', 'music.album.primary_release', 'Die in Your Arms'), ('1.FM Top 40', 'broadcast.content.artist', 'Boomtang Boys'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Cherish'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Peter Bjorn and John'), ('Record producer', 'common.topic.subjects', 'Steve Blevins'), ('Urban contemporary', 'broadcast.genre.content', '1Club.FM: V101'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Justin Timberlake'), ('Aaliyah', 'people.person.profession', 'Singer'), ('HitzRadio.com', 'common.topic.image', 'm.03vbp89'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mis-Teeq'), ('Heartbreaker', 'music.composition.composer', 'Xavier Smith'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Avril Lavigne'), ('m.0pc67ly', 'award.award_nomination.award', 'MTV Movie Award for Best Jaw Dropping Moment'), ('Justin Bieber', 'common.topic.webpage', 'm.0r90dwg'), ('My Worlds Acoustic', 'music.album.releases', 'My Worlds Acoustic'), ('m.0zg8bc1', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Leif Garrett', 'people.person.profession', 'Singer'), ('Rihanna', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('My Worlds: The Collection', 'music.album.primary_release', 'My Worlds: The Collection'), ('Justin Bieber', 'music.artist.album', 'Never Say Never: The Remixes'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('radioIO Todays POP', 'common.topic.article', 'm.045zxs7'), ('Juicy J', 'music.composer.compositions', 'Lolly'), ('Alicia Keys', 'common.topic.notable_types', 'Musical Artist'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber: Never Say Never', 'common.topic.webpage', 'm.0gfpb_k'), ('All Bad', 'music.recording.song', 'All Bad'), ('Stuart Ford', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Duffy'), ('Lil Jon', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.episode', \"Judge's House No. 2\"), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Writer'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('LeAnn Rimes', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fvqf', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Lil Jon', 'broadcast.artist.content', 'HitzRadio.com'), ('Miley Cyrus', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3bl'), ('Eminem', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bnz6'), ('Victoria Justice', 'people.person.profession', 'Actor'), ('Teyana', 'freebase.valuenotation.has_value', 'Height'), ('Yves Bole', 'music.artist.album', 'Rain'), ('Heartbreaker', 'music.composition.composer', 'Chef Tone'), ('School Boy Records', 'common.topic.article', 'm.0gmd15q'), ('#thatPower', 'common.topic.notable_for', 'g.1ydntw0_k'), ('Ashley Tisdale', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Journals', 'music.album.primary_release', 'Journals'), ('Soulja Boy', 'music.artist.genre', 'Hip hop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ray J'), ('m.0yrkc0l', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jay-Z'), ('1Club.FM: Power', 'broadcast.content.artist', 'Madonna'), ('Nicki Minaj', 'people.person.profession', 'Actor'), ('Madonna', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Jessica Simpson', 'music.artist.genre', 'Pop music'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhx1b'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Believe Acoustic', 'freebase.valuenotation.is_reviewed', 'Release type'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Contemporary R&B', 'music.genre.parent_genre', 'Rhythm and blues'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Aaliyah'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me (album version)'), ('Linkin Park', 'broadcast.artist.content', '1.FM Top 40'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ricky Nelson', 'base.icons.icon.icon_genre', 'Teen idol'), ('50 Cent', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Tommy Sands', 'common.topic.notable_types', 'Musical Artist'), ('Right Here', 'music.recording.artist', 'Justin Bieber'), ('m.0z8qx3w', 'award.award_honor.ceremony', '2010 Teen Choice Awards'), ('m.0njhxd_', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Avril Lavigne', 'broadcast.artist.content', 'Hot Wired Radio'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvpj'), ('David Cassidy', 'music.artist.genre', 'Pop music'), ('Yves Bole', 'music.artist.album', 'Moving On'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'JoJo'), ('m.0yrkr34', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Raekwon', 'people.person.profession', 'Actor'), ('Believe', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Black Eyed Peas', 'music.artist.genre', 'Synthpop'), ('Disco', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gj1yzm'), ('Diplo', 'people.person.gender', 'Male'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0jvgmxc'), ('Runaway Love (remix)', 'music.recording.releases', 'Runaway Love (remix)'), ('Under the Mistletoe', 'music.album.genre', 'Rhythm and blues'), ('P!nk', 'people.person.profession', 'Musician'), ('m.0gbm3bs', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Maroon 5', 'music.artist.genre', 'Rock music'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'common.topic.notable_types', 'Musical Recording'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Roots'), ('CL', 'music.artist.genre', 'Hip hop music'), ('Selena Gomez', 'people.person.profession', 'Actor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Coldplay'), ('Change Me', 'music.composition.composer', 'Justin Bieber'), ('Confident', 'common.topic.notable_types', 'Musical Recording'), ('One Less Lonely Girl', 'music.album.compositions', 'One Less Lonely Girl'), ('m.0430821', 'common.webpage.category', 'Topic Webpage'), ('m.0njhvsq', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Shaun Cassidy', 'common.topic.notable_types', 'Musical Artist'), ('Eurodance', 'music.genre.parent_genre', 'Electronic dance music'), ('m.0yqfny6', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('Juvenile', 'people.person.gender', 'Male'), ('m.0njhvsq', 'award.award_honor.award_winner', 'Justin Bieber'), ('Barry Weiss', 'people.person.profession', 'Record producer'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_for', 'g.1yl5wcrp8'), ('Gwen Stefani', 'music.artist.genre', 'Synthpop'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Mary J. Blige', 'broadcast.artist.content', 'FLOW 103'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074plm'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('GotRadio - RnB Classics', 'broadcast.content.artist', 'Michael Jackson'), ('Shaffer Smith', 'broadcast.artist.content', '181-beat'), ('Michael Jackson', 'people.person.profession', 'Dancer'), (\"O'Kelly Isley, Jr.\", 'music.artist.genre', 'Rhythm and blues'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ray J', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Indie rock', 'common.topic.subject_of', 'Alan Motley'), ('Jessie J', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('P!nk', 'broadcast.artist.content', 'HitzRadio.com'), ('Amerie', 'people.person.profession', 'Singer'), ('Duffy', 'music.artist.genre', 'Rock music'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Hip hop music'), ('The Black Eyed Peas', 'broadcast.artist.content', 'WildFMRadio.com'), ('#thatPOWER', 'common.topic.notable_for', 'g.12pkcwrn5'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Big Sean', 'music.artist.track', 'As Long as You Love Me'), ('Wait For a Minute', 'music.recording.song', 'Wait for a Minute'), ('Hold Tight', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'), ('Urban contemporary', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Dance music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Hellogoodbye'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Club Mix)'), ('First Dance', 'music.composition.lyricist', 'Dwight Reynolds'), ('Bad 25', 'common.topic.notable_types', 'Film'), ('Thought Of You', 'music.recording.releases', 'Believe'), ('SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Rock music'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Singer', 'people.profession.specializations', 'Gospel singer'), ('Alicia Keys', 'music.artist.genre', 'Rhythm and blues'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('All That Matters', 'common.topic.notable_types', 'Musical Album'), ('K-Ci & JoJo', 'people.person.gender', 'Male'), ('Nelly', 'broadcast.artist.content', '1Club.FM: Power'), ('Teen idol', 'common.topic.webpage', 'm.09wsj7g'), ('Ashley Tisdale', 'music.artist.genre', 'Teen pop'), ('Recovery', 'music.composition.composer', 'James Giannos'), ('Taylor Swift', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('Jordin Sparks', 'music.artist.genre', 'Teen pop'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Athan Grace'), ('Katy Perry', 'people.person.profession', 'Singer-songwriter'), ('m.0gxnnzy', 'celebrities.romantic_relationship.celebrity', 'Justin Bieber'), ('Michael Jackson', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sean Paul'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0ghz3d6'), ('Hikaru Utada', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Miami', 'common.topic.notable_types', 'City/Town/Village'), ('m.0t4s_bn', 'award.award_honor.award_winner', 'Justin Bieber'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0_vw6nn', 'award.award_nomination.nominated_for', 'All That Matters'), ('Linkin Park', 'music.artist.genre', 'Electronic music'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Pray', 'music.composition.composer', 'Adam Messinger'), ('Hold Tight', 'music.composition.composer', 'Justin Bieber'), ('Victoria Justice', 'people.person.languages', 'English Language'), ('Stephen Melton', 'people.person.profession', 'Singer-songwriter'), ('Nelly Furtado', 'music.artist.genre', 'Rock music'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Aqualung'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Big R Radio - Top 40 Hits', 'common.topic.notable_types', 'Broadcast Content'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Outkast'), ('Bryan-Michael Cox', 'people.person.profession', 'Singer'), ('Urban contemporary', 'broadcast.genre.content', '181-rnb'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.0v90p2b'), ('London', 'location.location.time_zones', 'Eastern Time Zone'), ('m.0d33hsj', 'celebrities.friendship.friend', 'Christian Beadles'), ('Hot Wired Radio', 'broadcast.content.artist', 'Lady Gaga'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('This is Justin Bieber', 'film.film.language', 'English Language'), ('All Bad', 'common.topic.notable_types', 'Musical Album'), ('The Notorious B.I.G.', 'people.person.gender', 'Male'), ('Teen idol', 'base.icons.icon_genre.icons', 'Leif Garrett'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Jay-Z'), ('Madonna', 'music.artist.genre', 'Electronic dance music'), ('Gwen Stefani', 'music.artist.genre', 'Rock music'), ('Ciara', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('As Long as You Love Me', 'music.composition.composer', 'Nasri'), ('William Orbit', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lifehouse'), ('Verse Simmonds', 'music.composer.compositions', 'Confident'), ('Fabolous', 'freebase.valuenotation.has_value', 'Parents'), ('m.0zg2qjr', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('Savan Kotecha', 'freebase.valuenotation.has_value', 'Parents'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Mary J. Blige'), ('Baby', 'music.music_video.artist', 'Ludacris'), ('m.0ncpx3v', 'organization.leadership.organization', 'The Island Def Jam Music Group'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Flo Rida'), ('Eminem', 'broadcast.artist.content', 'Smoothbeats'), ('Drake', 'music.artist.genre', 'Hip hop music'), ('Nick Jonas', 'people.person.profession', 'Actor'), ('Miley Cyrus', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Eenie Meenie', 'music.composition.composer', 'Justin Bieber'), ('Eminem', 'people.person.profession', 'Record producer'), ('Playback Singer', 'people.profession.specialization_of', 'Singer'), ('Pattie Mallette', 'people.person.children', 'Justin Bieber'), ('The Black Eyed Peas', 'broadcast.artist.content', '.977 The Hits Channel'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Dub)'), ('Blackstreet', 'broadcast.artist.content', '.977 The Hits Channel'), ('PG (USA)', 'film.content_rating.country', 'United States of America'), ('Sean Kingston', 'people.person.profession', 'Musician'), ('L.A. Reid', 'people.person.nationality', 'United States of America'), ('m.0vp800w', 'music.recording_contribution.album', 'All Around the World'), ('Baby', 'music.album.contributor', 'm.0vmyv4w'), ('Little Bird', 'music.recording.featured_artists', 'Justin Bieber'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Akon'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.appearance_type', 'Host'), ('PYD', 'music.album.release_type', 'Single'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Freddy Fader'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.broadcast', 'Emphatic Radio.com! - 24kbps Stream'), ('Justin Bieber', 'celebrities.celebrity.sexual_relationships', 'm.0gxnnzy'), ('HitzRadio.com', 'broadcast.content.artist', 'Jurassic 5'), ('Chris Brown', 'music.artist.genre', 'Pop music'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fuel'), ('Ashlee Simpson', 'people.person.profession', 'Singer-songwriter'), ('The Isley Brothers', 'broadcast.artist.content', 'radioIO Classic RNB'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.artist', 'Justin Bieber'), ('As Long as You Love Me', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'Home to Mama'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Jagged Edge'), ('Justin Timberlake', 'people.person.profession', 'Actor'), ('Justin Bieber', 'music.artist.album', 'Roller Coaster'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010h5lzt'), ('Lil Jon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jay-Z', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), (\"Bill O'Dowd\", 'film.producer.film', \"Justin Bieber's Believe\"), ('Leif Garrett', 'music.artist.genre', 'Disco'), ('m.0n4rmg7', 'award.award_honor.award_winner', 'Justin Bieber'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Wideboys Dub)'), ('Hot Wired Radio', 'broadcast.content.broadcast', 'Hot Wired Radio - 128kbps Stream'), ('As Long As You Love Me (Audien dubstep edit)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.artist.album', 'Boyfriend'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Dany Brillant'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d_'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gh0fdt'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Boyfriend', 'award.award_winning_work.awards_won', 'm.0yrkc0l'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Next to You', 'music.single.versions', 'Next to You'), ('m.0gxnp5d', 'base.popstra.hangout.business_location', 'Selena Gomez'), ('Jordan Pruitt', 'music.artist.genre', 'Rhythm and blues'), ('Bruce Dickinson', 'fictional_universe.fictional_character.gender', 'Male'), ('R. Kelly', 'music.artist.genre', 'Rhythm and blues'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('My World', 'music.album.release_type', 'EP'), ('Concert tour', 'freebase.type_profile.strict_included_types', 'Topic'), ('Michael Jackson', 'music.artist.genre', 'Disco'), ('Elvis Presley', 'people.person.gender', 'Male'), ('181-beat', 'broadcast.content.artist', 'Chris Brown'), ('m.0r90dwg', 'common.webpage.topic', 'Justin Bieber'), ('Benny Blanco', 'music.artist.genre', 'Synthpop'), ('Justin Bieber: Just Getting Started', 'book.book.editions', 'Justin Bieber: Just Getting Started'), ('All Around the World', 'music.recording.artist', 'Justin Bieber'), ('Journals', 'music.album.release_type', 'Album'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Wont Stop (feat. Justin Bieber)', 'common.topic.notable_for', 'g.1s052gdth'), ('Record producer', 'base.descriptive_names.names.descriptive_name', 'm.010f2m57'), ('Actor', 'music.special_music_video_performance_type.special_music_video_performances', 'm.0z3vx9q'), ('m.0kt8f87', 'people.place_lived.person', 'Justin Bieber'), ('Pearl Jam', 'broadcast.artist.content', '1.FM Top 40'), ('Jeremy Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.01053qzf'), ('Coldplay', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Hip hop music', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.artist', 'Panic! at the Disco'), ('Sean Combs', 'common.topic.notable_types', 'Musical Artist'), ('Pray', 'common.topic.notable_for', 'g.12h2wqwc4'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Jay-Z'), ('Avery', 'people.person.profession', 'Singer-songwriter'), ('GotRadio - RnB Classics', 'broadcast.content.artist', 'The Isley Brothers'), ('1Club.FM: Channel One', 'common.topic.webpage', 'm.043082k'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (DJ Laszlo Body Rock Instrumental)'), ('m.0vmyv4w', 'music.recording_contribution.album', 'Baby'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('1Club.FM: V101', 'broadcast.content.artist', 'Jagged Edge'), ('Recovery', 'music.recording.artist', 'Justin Bieber'), ('Pray', 'common.topic.notable_types', 'Composition'), ('Estelle', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'Leona Lewis'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rob Thomas', 'music.artist.genre', 'Dance music'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Nelly'), ('Katy Perry', 'music.artist.genre', 'Europop'), ('Shaggy', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.012bm5cg', 'celebrities.rivalry.rival', 'Yves Bole'), ('R. Kelly', 'people.person.profession', 'Musician'), ('m.0gbm3cp', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('1Club.FM: Power', 'broadcast.content.artist', 'Twista'), ('J. Holiday', 'common.topic.notable_types', 'Musical Artist'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhvsq'), ('Scooter Braun', 'freebase.valuenotation.is_reviewed', 'Height'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Montell Jordan'), ('Thought Of You', 'music.composition.recordings', 'Thought Of You'), ('m.0gxnp72', 'base.popstra.support.supported_organization', 'Pattie Mallette'), ('Jessie J', 'influence.influence_node.influenced_by', 'Yves Bole'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Drake', 'people.person.gender', 'Male'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juno Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0njvvj_'), ('Baby', 'music.album.genre', 'Eurodance'), ('Scooter Braun', 'film.person_or_entity_appearing_in_film.films', 'm.0101fsyr'), ('Kanye West', 'broadcast.artist.content', 'Hot 108 Jamz'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjhc6c'), ('Rob Thomas', 'common.topic.notable_types', 'Musical Artist'), ('The Pussycat Dolls', 'music.artist.genre', 'Contemporary R&B'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Beauty and a Beat', 'music.recording.releases', 'Believe'), ('Jason Mraz', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Vanessa Hudgens', 'people.person.gender', 'Female'), ('We Are the World 25 for Haiti', 'common.topic.notable_types', 'Musical Album'), ('Pattie Mallette', 'people.person.sibling_s', 'm.0wfn4pm'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('m.0n1ykxp', 'award.award_honor.honored_for', 'Baby'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0108gy4p'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Dyce'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Dub Mix)'), ('m.0wjhc6c', 'award.award_honor.ceremony', '2013 Teen Choice Awards'), ('.977 The Hits Channel', 'broadcast.content.genre', 'Pop music'), ('Ja Rule', 'broadcast.artist.content', 'Smoothbeats'), ('m.0njvvj_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k8y'), ('Justin Timberlake', 'music.artist.genre', 'Hip hop music'), ('Wait for a Minute', 'music.composition.composer', 'Tyga'), ('Image', 'type.property.schema', 'Topic'), ('m.0j8mhs_', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('m.0zbg2kw', 'award.award_nomination.ceremony', 'American Music Awards of 2010'), ('Synthpop', 'music.genre.parent_genre', 'Electronic dance music'), ('m.0z8qqh5', 'award.award_nomination.award', 'Teen Choice Award for Choice Music: Album Pop'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Never Say Never: The Remixes', 'common.topic.notable_for', 'g.125980jf9'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Boys Like Girls'), ('Jaxon Bieber', 'people.person.place_of_birth', 'Canada'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Ontario', 'base.biblioness.bibs_location.country', 'Canada'), ('Height', 'rdf-schema#range', 'Floating Point Number'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Hip hop music'), ('Dance music', 'broadcast.genre.content', 'C9 Radio'), ('Beauty and a Beat (Remixes)', 'common.topic.notable_for', 'g.126tkfc2x'), ('Carrie Underwood', 'people.person.nationality', 'United States of America'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Right Here', 'music.composition.recordings', 'Right Here'), ('Bigger', 'music.composition.lyricist', 'Dapo Torimiro'), ('m.0vp7mrq', 'music.recording_contribution.album', 'Runaway Love (remix)'), ('Justin Bieber', 'broadcast.artist.content', 'radioIO Todays POP'), ('Taylor Swift', 'music.artist.genre', 'Dance-pop'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0gbcs1_', 'tv.tv_guest_role.episodes_appeared_in', 'March 16, 2010'), ('Sean Kingston', 'people.person.profession', 'Record producer'), ('My Worlds Acoustic', 'music.album.genre', 'Pop music'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Justin Bieber'), ('Christmas in Washington', 'common.topic.notable_types', 'Film'), ('Justin Timberlake', 'people.person.profession', 'Record producer'), ('Right Here (featuring Drake)', 'music.recording.artist', 'Justin Bieber'), ('Jordin Sparks', 'people.person.profession', 'Model'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cajjmere Wray'), ('Jeremy Bieber', 'people.person.nationality', 'Canada'), ('HitzRadio.com', 'broadcast.content.artist', 'My Chemical Romance'), ('Janet Jackson', 'broadcast.artist.content', 'DeeGay'), ('C1', 'people.person.profession', 'Musician'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5k7g'), ('Jaden Smith', 'music.featured_artist.recordings', 'Never Say Never'), ('Aaliyah', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_721l'), ('Rihanna', 'music.artist.genre', 'Urban contemporary'), ('m.0w5l5h1', 'education.education.student', 'Justin Bieber'), ('Justin Bieber', 'music.artist.album', 'Pray'), ('Avril Lavigne', 'people.person.profession', 'Model'), ('Adam Messinger', 'music.composer.compositions', 'Pray'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Bigger', 'music.recording.artist', 'Justin Bieber'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mariah Carey'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('Chef Tone', 'people.person.gender', 'Male'), ('Beauty and a Beat (DJ Laszlo Body Rock Club Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Hoobastank'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'Dan Cutforth'), ('Juelz Santana', 'music.artist.genre', 'Hip hop music'), ('Will i Am', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'music.featured_artist.albums', 'Lolly'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Fergie'), ('Jayceon Terrell Taylor', 'music.artist.genre', 'Contemporary R&B'), ('Beautiful', 'music.recording.artist', 'Justin Bieber'), ('One Time', 'music.album.releases', 'One Time'), ('m.0_sxby0', 'award.award_nomination.award', 'Shorty Award for Music'), ('Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jessie J', 'people.person.gender', 'Female'), ('m.0b497pt', 'common.webpage.topic', 'Record producer'), ('School Boy Records', 'music.record_label.artist', 'Justin Bieber'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Carrie Underwood', 'people.person.profession', 'Actor'), ('Jay-Z', 'people.person.profession', 'Film Producer'), ('Diplo', 'music.artist.genre', 'Electronic dance music'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Spanish Language', 'language.human_language.countries_spoken_in', 'United States of America'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Christina Aguilera', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0dm4cqr'), ('All Around The World (featuring Ludacris)', 'music.recording.featured_artists', 'Ludacris'), ('London', 'common.topic.notable_types', 'City/Town/Village'), ('Tupac Shakur', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'CKY'), ('HitzRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Gwen Stefani', 'broadcast.artist.content', '1Club.FM: Channel One'), ('P!nk', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Benny Blanco', 'music.artist.genre', 'Rhythm and blues'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Shaggy'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Tupac Shakur', 'freebase.valuenotation.has_no_value', 'Children'), ('Christina Milian', 'music.artist.genre', 'Dance music'), ('m.0jzrrqs', 'common.topic.notable_types', 'Location'), ('m.0tkqqgg', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Nelly', 'broadcast.artist.content', '181-beat'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw1tn'), ('Rihanna', 'music.artist.genre', 'Reggae'), ('Deborah Lurie', 'people.person.profession', 'Record producer'), ('Music Producer', 'common.topic.subjects', 'POPPMusic.net'), ('Record producer', 'common.topic.webpage', 'm.0b48fj5'), ('m.0njdq6k', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Nelly', 'people.person.profession', 'Singer'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Wideboys Radio Mix)'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv7d'), ('Trey Songz', 'people.person.languages', 'English Language'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Lady Gaga', 'people.person.profession', 'Singer-songwriter'), ('m.0101fvq6', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'music.artist.album', 'Eenie Meenie'), ('Santana', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Jane Lipsitz', 'people.person.nationality', 'United States of America'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('#thatPOWER', 'music.album.compositions', '#thatPower'), ('Adam Messinger', 'people.person.profession', 'Musician'), ('Usher', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Sean Kingston', 'people.person.place_of_birth', 'Miami'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Contemporary R&B'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Profession'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Sean Paul'), ('Estelle', 'people.person.profession', 'Actor'), ('m.0pc670l', 'award.award_honor.award', 'MTV Movie Award for Best Jaw Dropping Moment'), ('Ernie Isley', 'freebase.valuenotation.has_value', 'Parents'), ('1.FM Top 40', 'broadcast.content.artist', 'Kylie Minogue'), ('My World 2.0', 'music.album.artist', 'Justin Bieber'), ('Beautiful', 'music.composition.recordings', 'Beautiful'), ('m.0101fv4c', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Ciara', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'music.artist.genre', 'Contemporary R&B'), ('American Music Award for Favorite Pop/Rock Male Artist', 'award.award_category.winners', 'm.0njdq6k'), ('Teen idol', 'base.icons.icon_genre.icons', 'Donny Osmond'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Award category'), ('Teen idol', 'base.icons.icon_genre.icons', 'Shaun Cassidy'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Twista', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('PowerHitz', 'broadcast.content.artist', 'Leona Lewis'), ('Timbaland', 'people.person.profession', 'Record producer'), ('Mariah Carey', 'people.person.gender', 'Female'), ('Big R Radio - The Hawk', 'broadcast.content.artist', '50 Cent'), ('m.0kt8f87', 'people.place_lived.location', 'Stratford'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_706c'), ('Akon', 'broadcast.artist.content', 'HitzRadio.com'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Rubyhorse'), ('Jay Cassidy', 'people.person.gender', 'Male'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('CL', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('HitzRadio.com', 'broadcast.content.artist', 'P!nk'), ('Pray', 'music.composition.recordings', 'Pray'), ('m.0101fs_z', 'film.personal_film_appearance.person', 'Jeremy Bieber'), ('Tupac Shakur', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Justin Timberlake'), ('Pray', 'music.composition.composer', 'Justin Bieber'), ('Vanessa Hudgens', 'music.artist.genre', 'Dance-pop'), ('Writer', 'common.topic.notable_types', 'Profession'), ('Red Hot Chili Peppers', 'music.artist.genre', 'Reggae'), ('Spouse', 'type.property.schema', 'Marriage'), ('Akon', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0hmvx8s', 'people.marriage.spouse', 'Jeremy Bieber'), ('Will i Am', 'music.artist.genre', 'Contemporary R&B'), ('Ronski Speed', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_6zk4'), ('Justin Bieber', 'music.artist.album', 'Hold Tight'), ('Heartbreaker', 'common.topic.notable_for', 'g.12z651m29'), ('Live My Life', 'music.composition.recordings', 'Live My Life (Party Rock remix)'), ('Contemporary R&B', 'common.topic.image', 'Contemporary R&B singer Mary J. Blige performs on the National Mall during the “NFL Kickoff Live 2003” Concert'), ('Britney Spears', 'people.person.profession', 'Artist'), ('Lil Wayne', 'people.person.profession', 'Musician'), ('1Club.FM: V101', 'broadcast.content.artist', 'Whitney Houston'), ('Sean Kingston', 'music.artist.genre', 'Pop music'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Children'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Zendaya: Behind the Scenes', 'film.film.country', 'United States of America'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Musical Artist', 'freebase.documented_object.documentation', 'm.01z0mrz'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Avril Lavigne', 'people.person.profession', 'Musician'), ('JoJo', 'music.artist.genre', 'Hip hop music'), ('m.012bm4v7', 'celebrities.friendship.friend', 'Rita Ora'), ('Live My Life', 'music.album.releases', 'Live My Life'), ('Janet Jackson', 'people.person.profession', 'Record producer'), ('m.0y5t8gm', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Pop music', 'music.genre.parent_genre', 'Dance music'), ('All Around the World', 'music.recording.song', 'All Around The World'), ('The Isley Brothers', 'music.artist.genre', 'Rhythm and blues'), ('Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Lady Gaga', 'broadcast.artist.content', 'radioIO Todays POP'), ('Lupe Fiasco', 'music.artist.origin', 'Chicago'), ('Ice Cube', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: Power', 'broadcast.content.artist', 'Fergie'), ('m.0nhfd4m', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Singer-songwriter', 'people.profession.corresponding_type', 'Musical Artist'), ('PowerHitz', 'broadcast.content.artist', 'Lil Wayne'), ('Don Henley', 'people.person.nationality', 'United States of America'), ('radioIO Todays RNB', 'common.topic.image', 'RadioIO.png'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0yqfny6'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_99fs'), ('The Roots', 'music.artist.genre', 'Hip hop music'), ('m.0gwhmhm', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Don Henley', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('As Long As You Love Me', 'music.recording.featured_artists', 'Big Sean'), ('Bad Day', 'music.album.releases', 'Bad Day'), ('Big R Radio - The Hawk', 'common.topic.image', 'Big R radio'), ('Jessie J', 'freebase.valuenotation.has_no_value', 'Children'), ('Janet Jackson', 'music.artist.label', 'Island Records'), ('m.0101ft2j', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('RedOne', 'music.artist.genre', 'Dance music'), ('Right Here', 'music.album.release_type', 'Single'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Gaga'), ('Miami Beach', 'common.topic.notable_types', 'City/Town/Village'), ('Never Say Never', 'common.topic.notable_types', 'Musical Recording'), ('Teen idol', 'base.icons.icon_genre.icons', 'Davy Jones'), ('m.0njw59_', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Bryan-Michael Cox', 'music.lyricist.lyrics_written', 'Never Let You Go'), ('Bigger', 'music.composition.composer', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'P!nk'), ('radioIO Todays POP', 'broadcast.content.genre', 'Rock music'), ('Lady Gaga', 'freebase.valuenotation.has_no_value', 'Children'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Children'), ('Next to You', 'music.album.artist', 'Chris Brown'), ('Tricky Stewart', 'music.artist.genre', 'Hip hop music'), ('R. Kelly', 'people.person.nationality', 'United States of America'), ('Cory Gunz', 'music.artist.genre', 'Contemporary R&B'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('WildFMRadio.com', 'broadcast.content.artist', 'Rihanna'), ('m.0ng_k21', 'film.personal_film_appearance.film', 'Christmas in Washington'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Shaffer Smith', 'people.person.profession', 'Record producer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jeff Klein'), ('Redfoo', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.genre', 'Rap music'), ('Beauty and a Beat', 'music.album.release_type', 'Single'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beautiful', 'common.topic.notable_for', 'g.126t625wd'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Bisbetic Instrumental)'), ('Willa Ford', 'people.person.nationality', 'United States of America'), ('Everlast', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'film.actor.film', 'm.0jztshx'), ('As Long as You Love Me', 'music.composition.language', 'English Language'), ('1.FM Top 40', 'broadcast.content.artist', 'Bryan Adams'), ('#Thatpower', 'music.recording.canonical_version', '#thatPOWER'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('My Worlds Acoustic', 'common.topic.notable_types', 'Musical Album'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0gbmnv1', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('Leonardo DiCaprio', 'freebase.valuenotation.has_no_value', 'Children'), ('Sia Furler', 'freebase.valuenotation.is_reviewed', 'Place of birth'), (\"Bill O'Dowd\", 'people.person.profession', 'Film Producer'), ('Teen pop', 'music.genre.parent_genre', 'K-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0115qhzk'), ('Baby', 'music.composition.recordings', 'Baby'), ('Beauty And A Beat', 'common.topic.notable_for', 'g.1258kxk_f'), ('Hot Wired Radio', 'broadcast.content.artist', 'Shop Boyz'), ('Lil Jon', 'music.artist.genre', 'Hip hop music'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Hot Wired Radio', 'broadcast.content.artist', 'Gym Class Heroes'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_6_81'), ('m.0v1lwt2', 'tv.tv_guest_role.special_performance_type', 'Cameo appearance'), ('m.0y4tdml', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0gctwjk', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0_grmr_', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), ('Kelis', 'broadcast.artist.content', '1Club.FM: Power'), ('Montell Jordan', 'people.person.profession', 'Singer-songwriter'), ('m.0v30sv7', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'music.recording.tracks', 'Live My Life (Jaywalker remix)'), ('Baby', 'common.topic.article', 'm.09v3gb8'), ('Beauty and a Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Runaway Love (remix)', 'music.recording.artist', 'Kanye West'), ('Rihanna', 'music.artist.genre', 'Pop music'), ('Jayceon Terrell Taylor', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordan Francis', 'music.artist.genre', 'Hip hop music'), ('Contemporary R&B', 'broadcast.genre.content', 'HitzRadio.com'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'common.topic.notable_types', 'Musical Recording'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Contemporary R&B'), ('m.0y84ynj', 'award.award_nomination.award_nominee', 'Will i Am'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Parents'), ('All Bad', 'common.topic.notable_types', 'Musical Recording'), ('Indie rock', 'common.topic.subjects', 'Singer-songwriter'), ('Dr. Dre', 'broadcast.artist.content', 'JellyRadio.com'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Snoop Dogg', 'people.person.profession', 'Film Producer'), (\"Destiny's Child\", 'music.artist.genre', 'Hip hop music'), ('Lolly', 'music.album.releases', 'Lolly'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Hot 108 Jamz'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Bad Boys Blue'), ('Musician', 'common.topic.subject_of', 'Artfarm'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ludacris'), ('Love Me', 'music.album.artist', 'Justin Bieber'), ('Adam Messinger', 'music.composer.compositions', 'Never Say Never'), ('Justin Bieber', 'common.topic.notable_for', 'g.1259ft7c9'), ('m.0z8t2dy', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('The Roots', 'common.topic.notable_types', 'Musical Artist'), ('Beauty and a Beat (Wideboys Club Mix)', 'common.topic.notable_types', 'Musical Recording'), ('1.FM Top 40', 'broadcast.content.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Rap music'), (\"Kids' Choice Award for Favorite Song\", 'award.award_category.winners', 'm.0sgk_cw'), ('Beauty And A Beat', 'music.composition.composer', 'Nicki Minaj'), ('1.FM Top 40', 'broadcast.content.artist', 'The Hoosiers'), ('Mariah Carey', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0101fszs', 'film.personal_film_appearance.person', 'Pattie Mallette'), ('My World 2.0', 'music.album.genre', 'Contemporary R&B'), ('m.0yrhhqv', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrlqp1', 'freebase.valuenotation.has_value', 'Start Date'), ('Roller Coaster', 'music.recording.artist', 'Justin Bieber'), ('Shaffer Smith', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Pattie Mallette', 'common.topic.notable_for', 'g.125gnww48'), ('Profession', 'rdf-schema#domain', 'Person'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Janet Jackson'), ('Sean Kingston', 'freebase.valuenotation.is_reviewed', 'Gender'), ('PYD', 'music.album.primary_release', 'PYD'), ('JoJo', 'music.artist.genre', 'Dance-pop'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Top 40'), ('m.0njgyk4', 'award.award_honor.ceremony', 'American Music Awards of 2010'), ('Location', 'type.type.properties', 'People born here'), ('The Isley Brothers', 'music.artist.genre', 'Disco'), ('Justin Bieber', 'common.topic.webpage', 'm.07lkzw7'), ('Clay Aiken', 'people.person.gender', 'Male'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Keyshia Cole', 'people.person.profession', 'Actor'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Roller Coaster', 'music.album.release_type', 'Single'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'Paramount Pictures'), ('m.0101ftm6', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Nelly', 'people.person.gender', 'Male'), ('Start Date', 'rdf-schema#range', 'Date/Time'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Blackstreet'), ('Britney Spears', 'people.person.profession', 'Singer-songwriter'), ('Die in Your Arms', 'music.album.release_type', 'Single'), ('Live My Life', 'music.composition.recorded_as_album', 'Live My Life'), ('Young Hollywood Award for Newcomer of the Year', 'award.award_category.winners', 'm.0yr9c1k'), ('Aaliyah', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Anastacia', 'music.artist.genre', 'Electronic dance music'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Trick Daddy', 'broadcast.artist.content', '1Club.FM: Power'), ('WildFMRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('Drake', 'people.person.nationality', 'Canada'), ('Rudolph Isley', 'music.composer.compositions', 'Bad Day'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Canada', 'location.country.first_level_divisions', 'Ontario'), ('Artist', 'people.profession.specializations', 'Dancer'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Mary J. Blige'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Amerie', 'music.artist.genre', 'Rhythm and blues'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Parents'), ('One Less Lonely Girl', 'music.album.genre', 'Rhythm and blues'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.composer.compositions', 'All That Matters'), ('Billboard Music Award for Top Social Artist', 'award.award_category.winners', 'm.010lkp2z'), ('Nick Jonas', 'people.person.profession', 'Musician'), ('Shaffer Smith', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Teen Choice Award for Choice TV Villain', 'award.award_category.winners', 'm.0z898w6'), ('m.0yrjvlh', 'award.award_honor.award_winner', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'J. Holiday'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Ludacris'), ('m.0yrktlv', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Simple Plan'), ('Hot Wired Radio', 'broadcast.content.artist', 'Cascada'), ('Carrie Underwood', 'common.topic.notable_types', 'Musical Artist'), ('181-beat', 'broadcast.content.artist', 'Leona Lewis'), ('Deborah Lurie', 'freebase.valuenotation.has_value', 'Parents'), ('Cory Gunz', 'music.artist.genre', 'Rhythm and blues'), ('Frank Ocean', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Official website'), (\"Bill O'Dowd\", 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World', 'music.album.genre', 'Synthpop'), ('MTV Europe Music Award for Best Push Artist', 'award.award_category.winners', 'm.0njw257'), ('Boyfriend', 'music.album.artist', 'Justin Bieber'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Aaliyah'), ('1Club.FM: Mix 106', 'broadcast.content.broadcast', '1Club.FM: Mix 106 - 64kbps Stream'), ('Benny Blanco', 'people.person.profession', 'Musician'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Justin Bieber'), ('David Cassidy', 'people.person.profession', 'Writer'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Savan Kotecha', 'music.artist.label', 'Island Records'), ('CL', 'people.person.profession', 'Dancer'), ('The Black Eyed Peas', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber', 'music.group_member.membership', 'm.011m26pv'), ('m.0njwqrb', 'award.award_honor.award', 'Queen Elizabeth II Diamond Jubilee Medal'), ('1Club.FM: Channel One', 'common.topic.notable_types', 'Broadcast Content'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Aaliyah'), ('Les Coulisses des Golden Globes', 'common.topic.notable_types', 'Film'), ('1Club.FM: Power', 'broadcast.content.artist', 'Nelly'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjgqck'), ('My Worlds Acoustic', 'common.topic.image', '220px-Myworldsacoustic.jpg'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Whitney Houston', 'people.person.profession', 'Actor'), ('Bigger', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Trey Songz', 'people.person.profession', 'Record producer'), ('Right Here', 'common.topic.notable_types', 'Canonical Version'), ('Chris Brown', 'people.person.profession', 'Singer'), ('One Less Lonely Girl', 'music.album.artist', 'Justin Bieber'), ('Fergie', 'music.artist.genre', 'Contemporary R&B'), ('Boyfriend', 'music.composition.recorded_as_album', 'Boyfriend'), ('Recovery', 'common.topic.notable_for', 'g.1yl5xlb58'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'people.person.places_lived', 'm.0gxnny8'), ('Spouse (or domestic partner)', 'type.property.expected_type', 'Marriage'), ('Official website', 'rdf-schema#range', 'URI'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_436'), ('Jessie J', 'music.artist.genre', 'Pop music'), ('m.064xzb1', 'common.webpage.category', 'Official Website'), ('Dr. Dre', 'people.person.nationality', 'United States of America'), ('Love Never Felt So Good', 'music.composition.form', 'Song'), ('Hip hop music', 'broadcast.genre.content', 'Hot Wired Radio'), ('m.0v90p2b', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_70rd'), ('Record producer', 'common.topic.subject_of', 'POPPMusic.net'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Rihanna'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Top 40'), ('2011 MTV Movie Awards', 'award.award_ceremony.awards_presented', 'm.0pc670l'), ('m.03hs4ny', 'common.webpage.topic', 'BeirutNights.com Radio'), ('Beyoncé Knowles', 'people.person.profession', 'Singer-songwriter'), ('m.0vb6hhj', 'award.award_honor.ceremony', '2013 Billboard Music Awards'), ('iJustine', 'celebrities.celebrity.celebrity_rivals', 'm.012bm5cg'), ('Beauty and a Beat (Steven Redant Beauty and The Vocal Dub Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'iJustine'), ('Usher', 'broadcast.artist.content', 'PowerHitz'), ('Next to You', 'music.album.album_content_type', 'Studio album'), ('All Around The World', 'common.topic.article', 'm.0j_3rq4'), ('Believe Acoustic', 'music.album.releases', 'Believe Acoustic'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Chris Brown', 'people.person.profession', 'Singer-songwriter'), ('Katy Perry: Part of Me', 'film.film.genre', 'Music'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0h4yhbb'), ('Cris Cab', 'people.person.nationality', 'United States of America'), ('m.0ng_j6d', 'film.personal_film_appearance.film', 'This is Justin Bieber'), ('Aaliyah', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('1.FM Top 40', 'broadcast.content.artist', 'Kelly Clarkson'), ('Singer', 'people.profession.specializations', 'Dramatic coloratura'), ('Daniel Bedingfield', 'common.topic.notable_types', 'Musical Artist'), ('m.0ywvh8k', 'award.award_honor.honored_for', 'justinbieber'), ('Beautiful', 'music.recording.artist', 'Carly Rae Jepsen'), ('Enrique Iglesias', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Shaggy'), ('Chris Brown', 'music.artist.genre', 'Rhythm and blues'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Saturdays'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('Martin Kierszenbaum', 'common.topic.notable_types', 'Record Producer'), ('Jordin Sparks', 'people.person.languages', 'English Language'), ('DMX', 'people.person.nationality', 'United States of America'), ('Baby', 'music.composition.recordings', 'Baby'), ('Wait for a Minute', 'music.composition.composer', 'Justin Bieber'), ('Lady Gaga', 'people.person.nationality', 'United States of America'), ('Live My Life', 'music.recording.song', 'Live My Life'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Confident', 'common.topic.notable_types', 'Musical Album'), ('My World', 'award.award_winning_work.awards_won', 'm.0z8s_wn'), ('Dance-pop', 'music.genre.parent_genre', 'K-pop'), ('Big Sean', 'music.featured_artist.albums', 'As Long as You Love Me'), ('Baby', 'music.recording.featured_artists', 'Ludacris'), ('Will i Am', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber', 'broadcast.artist.content', '1.FM Top 40'), ('Justin Bieber', 'music.artist.album', 'Journals'), ('Justin Timberlake', 'broadcast.artist.content', 'radioIO Todays POP'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Usher', 'music.featured_artist.recordings', 'Somebody to Love'), ('Will Smith', 'broadcast.artist.content', '1Club.FM: Power'), ('Christian Beadles', 'people.person.nationality', 'United States of America'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Christina Milian', 'people.person.profession', 'Model'), ('Justin Timberlake', 'music.artist.genre', 'Pop music'), ('Boyfriend (Dada Life remix)', 'music.recording.artist', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Killers'), ('Pop music', 'music.genre.parent_genre', 'World music'), ('m.0njhxzc', 'award.award_honor.award_winner', 'Justin Bieber'), ('Lolly', 'common.topic.notable_types', 'Composition'), ('Colbie Caillat', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Goo Goo Dolls'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Parents'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0v_98t5', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('As Long As You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('Gas Pedal', 'music.recording.artist', 'Iamsu!'), ('1Club.FM: Power', 'broadcast.content.genre', 'Pop music'), ('Sir Mix-a-Lot', 'common.topic.notable_types', 'Musical Artist'), ('Mistletoe', 'music.album.releases', 'Mistletoe'), ('Stephen Melton', 'common.topic.subjects', 'Pop music'), ('Janet Jackson', 'broadcast.artist.content', '1Club.FM: V101'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Height'), ('Hold Tight', 'common.topic.notable_types', 'Musical Album'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Children', 'type.property.expected_type', 'Person'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Thought Of You', 'common.topic.notable_types', 'Composition'), ('Chingy', 'broadcast.artist.content', 'radioIO Todays RNB'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'FLOW 103'), ('First Dance', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Gwen Stefani'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'P!nk'), ('2011 MTV Video Music Aid Japan', 'award.award_ceremony.awards_presented', 'm.0yrhrwc'), ('Donna Summer', 'people.person.profession', 'Singer'), ('Green Day', 'broadcast.artist.content', 'HitzRadio.com'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0430821', 'common.webpage.topic', '1Club.FM: Mix 106'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yqfny6'), ('m.0gbm3cx', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('American Music Awards of 2010', 'award.award_ceremony.awards_presented', 'm.0njdq6k'), ('Height', 'type.property.expected_type', 'Floating Point Number'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Brandon Blue Hamilton'), ('Pray', 'music.album.release_type', 'Single'), ('Marvin Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Gas Pedal', 'music.recording.featured_artists', 'Iamsu!'), ('FLOW 103', 'broadcast.content.artist', 'Snoop Dogg'), ('Blu Cantrell', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.featured_artist.recordings', 'g.1q6708pcc'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhxd_'), ('Lady Antebellum', 'broadcast.artist.content', '.977 The Hits Channel'), ('Will Smith', 'people.person.languages', 'English Language'), ('Snoop Dogg', 'people.person.nationality', 'United States of America'), ('Sean Combs', 'people.person.profession', 'Film Producer'), ('Jaden Smith', 'music.featured_artist.recordings', 'Never Say Never (acoustic)'), ('Teyana', 'music.artist.genre', 'Hip hop music'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Stu Fisher'), ('Fabolous', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Baby', 'music.album.featured_artists', 'Ludacris'), ('Beauty and a Beat (Remixes)', 'music.album.artist', 'Justin Bieber'), ('m.0v_70rd', 'tv.tv_guest_personal_appearance.episode', 'Semi-Final'), ('Boyfriend', 'common.topic.notable_types', 'Composition'), ('m.0njhxd_', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhx1b'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Beyoncé Knowles'), ('The Isley Brothers', 'music.artist.label', 'The Island Def Jam Music Group'), ('Nelly', 'broadcast.artist.content', 'HitzRadio.com'), ('Hit-Boy', 'freebase.valuenotation.has_value', 'Parents'), ('Soulja Boy', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Height'), ('Dany Brillant', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('JellyRadio.com', 'broadcast.content.artist', '112'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Wait For a Minute', 'music.album.releases', 'Wait For a Minute'), ('All Around the World', 'music.album.releases', 'All Around the World'), ('Singer', 'base.schemastaging.context_name.pronunciation', 'g.125_qgwzx'), ('m.012nv5gz', 'people.place_lived.person', 'Yves Bole'), ('Katy Perry', 'music.artist.genre', 'Rock music'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Brandy Norwood', 'broadcast.artist.content', '1Club.FM: V101'), ('Pattie Mallette', 'people.person.parents', 'Michael Mallette'), ('Beauty and a Beat (Steven Redant Beauty and The Club Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Pray', 'common.topic.notable_types', 'Musical Album'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Nelly Furtado'), ('United States Dollar', 'base.coinsdaily.coin_type.country', 'United States of America'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010nwpx1'), ('Composition', 'type.type.properties', 'Lyricist'), ('Yves Bole', 'influence.influence_node.influenced_by', 'Jessie J'), ('Enrique Iglesias', 'music.artist.origin', 'Miami'), ('justinbieber', 'internet.blog.blogger', 'Justin Bieber'), ('m.0pc670l', 'award.award_honor.honored_for', 'Justin Bieber: Never Say Never'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jrb'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audiobot remix)'), ('Never Say Never', 'music.single.versions', 'Never Say Never (acoustic)'), ('Big Sean', 'people.person.gender', 'Male'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Official website'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Height'), ('m.0gxnnwp', 'people.sibling_relationship.sibling', 'Jaxon Bieber'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rap music'), ('U Smile', 'common.topic.notable_types', 'Musical Album'), ('Singer-songwriter', 'common.topic.subject_of', 'Indie rock'), ('1Club.FM: 80s (Pop)', 'common.topic.notable_types', 'Broadcast Content'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'BWO'), ('m.0z8s_wn', 'award.award_honor.ceremony', 'NME Awards 2011'), ('Beauty and a Beat', 'music.album.album_content_type', 'Studio album'), ('Amerie', 'music.artist.genre', 'Contemporary R&B'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Leona Lewis'), ('Ashley Tisdale', 'music.artist.genre', 'Pop music'), ('m.0nfnx_3', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('m.0tk76mf', 'award.award_nomination.ceremony', 'Juno Awards of 2011'), ('Rita Ora', 'celebrities.celebrity.celebrity_friends', 'm.012bm3j9'), ('m.0z8s562', 'award.award_honor.award', 'NME Award for Worst Dressed'), ('Jayceon Terrell Taylor', 'people.person.gender', 'Male'), ('Mariah Carey', 'broadcast.artist.content', 'radioIO RNB Mix'), ('m.0z83x9l', 'award.award_honor.award_winner', 'Justin Bieber'), ('All Bad', 'music.album.primary_release', 'All Bad'), ('Khalil', 'common.topic.notable_types', 'Musical Artist'), ('Recovery', 'music.composition.composer', 'Justin Bieber'), ('Whitney Houston', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'), ('Britney Spears', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Hip hop music', 'broadcast.genre.content', 'HitzRadio.com'), ('K-Ci & JoJo', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Justin Bieber: Never Say Never', 'film.film.rating', 'G (USA)'), ('Sean Kingston', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Fabian', 'common.topic.notable_types', 'Musical Artist'), (\"Dick Clark's Primetime New Year's Rockin' Eve 2013\", 'common.topic.notable_types', 'Film'), ('Hot Wired Radio', 'broadcast.content.artist', 'Fergie'), ('m.0z1ndbl', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Beautiful', 'music.composition.lyricist', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrjvlh'), ('My Worlds Acoustic', 'music.album.genre', 'Contemporary R&B'), ('Actor', 'common.topic.notable_types', 'Profession'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjk4'), ('HitzRadio.com', 'broadcast.content.artist', 'Gwen Stefani'), ('Big Sean', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat', 'music.album.artist', 'Justin Bieber'), ('Heartbreaker', 'music.composition.composer', 'Maejor'), ('Little Bird', 'music.recording.canonical_version', 'Live My Life'), ('radioIO Todays RNB', 'broadcast.content.producer', 'Radioio'), ('Jason Mraz', 'broadcast.artist.content', 'HitzRadio.com'), (\"Destiny's Child\", 'broadcast.artist.content', '.977 The Hits Channel'), ('My World', 'music.album.genre', 'Rhythm and blues'), ('Big Sean', 'music.featured_artist.recordings', 'As Long as You Love Me'), ('m.0f0dwc4', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('The Pussycat Dolls', 'music.artist.genre', 'Dance-pop'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Usher', 'music.artist.genre', 'Hip hop music'), ('Runaway Love (remix)', 'music.recording.song', 'Runaway Love'), ('Singer', 'people.profession.part_of_professional_field', 'Singing'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Believe', 'common.topic.article', 'm.0j263y7'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'P!nk'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('50 Cent', 'broadcast.artist.content', 'HitzRadio.com'), ('Leonardo DiCaprio', 'people.person.languages', 'English Language'), ('m.0yrhrwc', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrhhqv', 'award.award_honor.ceremony', '2010 MTV Video Music Brazil'), ('Never Let You Go', 'music.recording.artist', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'August Rigo'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Kylie Minogue', 'people.person.gender', 'Female'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Carrie Underwood', 'broadcast.artist.content', 'Hot Wired Radio'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Anna Nalick'), ('Musician', 'base.lightweight.profession.specialization_of', 'Musicians and Singers'), ('Justin Timberlake', 'music.artist.genre', 'Teen pop'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mike Oldfield'), ('2013 Radio Disney Music Awards', 'award.award_ceremony.awards_presented', 'm.0y4tdml'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jane Lipsitz', 'film.director.film', 'Katy Perry: Part of Me'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Baby', 'common.topic.notable_types', 'Musical Recording'), ('Pray', 'music.composition.recordings', 'Pray'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Micko Larkin'), ('Diplo', 'people.person.profession', 'Record producer'), ('1.FM Top 40', 'broadcast.content.artist', 'Lighthouse Family'), ('Believe Tour', 'common.topic.article', 'm.0j_3p6y'), ('m.0tk76mf', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Heartbreaker', 'music.album.releases', 'Heartbreaker'), ('Justin Bieber', 'music.featured_artist.recordings', 'g.11btzxy8qx'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Jeremy Bieber', 'people.person.spouse_s', 'm.0hmvx8s'), ('m.03zb5cw', 'common.webpage.category', 'Topic Webpage'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Power'), ('Katy Perry', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0z87d3n', 'award.award_honor.award_winner', 'Justin Bieber'), ('Montell Jordan', 'people.person.profession', 'Actor'), ('JellyRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('Justin Bieber', 'music.artist.album', 'My World 2.0'), ('Sean Kingston', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.film_production_design_by', 'Tom E. Marzullo'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Steven Redant Beauty and The Club Mix)'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0y6dqtf'), ('Benny Blanco', 'people.person.nationality', 'United States of America'), ('Live My Life', 'common.topic.notable_for', 'g.126sk4331'), ('Chris Brown', 'music.artist.album', 'Next to You'), ('Miley Cyrus', 'music.artist.genre', 'Synthpop'), ('My Worlds Acoustic', 'common.topic.notable_for', 'g.125h3jvrd'), ('Roller Coaster', 'music.composition.composer', 'Josh Gudwin'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Donna Summer'), ('Hot Wired Radio', 'broadcast.content.artist', 'Buckcherry'), ('m.0gbm3fj', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Usher'), ('Sheryl Crow', 'music.artist.genre', 'Pop music'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Lolly', 'music.single.versions', 'Lolly'), ('1Club.FM: V101', 'broadcast.content.artist', 'Mariah Carey'), ('Actor', 'common.topic.subjects', 'Bleona'), ('As Long as You Love Me', 'music.composition.composer', 'Andre Lindal'), ('Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('50 Cent', 'music.artist.genre', 'Hip hop music'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njw1tn', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Ja Rule', 'broadcast.artist.content', 'HitzRadio.com'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), ('m.0v_706c', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fergie'), ('Juicy J', 'music.artist.genre', 'Hip hop music'), ('Chris Brown', 'broadcast.artist.content', '181-beat'), ('Runaway Love (remix)', 'music.album.releases', 'Runaway Love (remix)'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Gender'), ('CL', 'people.person.gender', 'Female'), ('Runaway Love (remix)', 'music.album.release_type', 'Single'), ('Usher', 'broadcast.artist.content', '1Club.FM: Power'), ('Disco', 'common.topic.notable_types', 'Musical genre'), ('Demi Lovato', 'music.artist.genre', 'Dance music'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Smoothbeats'), ('Juno Awards of 2011', 'award.award_ceremony.awards_presented', 'm.0njvtth'), ('Clay Aiken', 'people.person.profession', 'Actor'), ('Usher', 'music.featured_artist.recordings', 'First Dance'), ('Elvis Presley', 'people.person.profession', 'Singer'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Ciara', 'broadcast.artist.content', 'HitzRadio.com'), ('Juvenile', 'freebase.valuenotation.has_value', 'Children'), ('Yves Bole', 'people.person.sibling_s', 'm.0129jzth'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('m.09wwfnk', 'common.webpage.topic', 'Record producer'), ('Justin Bieber', 'book.author.works_written', 'Justin Bieber: First Step 2 Forever'), ('1.FM Top 40', 'broadcast.content.genre', 'Rock music'), ('World music', 'common.topic.notable_types', 'Musical genre'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.09wnj4l', 'common.webpage.topic', 'Teen idol'), ('Fergie', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Enrique Iglesias', 'music.artist.genre', 'Dance music'), ('Next to You', 'common.topic.notable_types', 'Musical Recording'), ('Martin Kierszenbaum', 'music.composer.compositions', 'Live My Life'), ('Will i Am', 'broadcast.artist.content', 'Hot Wired Radio'), ('My World 2.0', 'music.album.release_type', 'Album'), ('WildFMRadio.com', 'broadcast.content.artist', 'Ciara'), ('Record producer', 'common.topic.webpage', 'm.09xwqmn'), ('Pras', 'people.person.profession', 'Actor'), ('Chef Tone', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.produced_by', 'L.A. Reid'), ('Dan Cutforth', 'people.person.profession', 'Film Producer'), ('Ontario', 'base.aareas.schema.administrative_area.administrative_parent', 'Canada'), ('Runaway Love (remix)', 'music.album.contributor', 'm.0vp7mrq'), ('m.05sp3_n', 'business.company_name_change.company', 'The Island Def Jam Music Group'), ('Shaffer Smith', 'people.person.profession', 'Dancer'), ('All That Matters (video)', 'music.recording.artist', 'Justin Bieber'), ('Alanis Morissette', 'people.person.profession', 'Singer-songwriter'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Gloria Gaynor'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'The Pussycat Dolls'), ('Ja Rule', 'common.topic.notable_types', 'Musical Artist'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Beyoncé Knowles'), ('Sir Mix-a-Lot', 'people.person.profession', 'Record producer'), ('Justin Timberlake', 'music.artist.genre', 'Contemporary R&B'), ('Kelly Clarkson', 'music.artist.genre', 'Dance-pop'), ('Rihanna', 'music.artist.genre', 'Electronic dance music'), ('As Long as You Love Me (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Jennifer Lopez', 'people.person.languages', 'Spanish Language'), ('Beautiful', 'common.topic.notable_types', 'Composition'), ('Willa Ford', 'people.person.profession', 'Actor'), ('Usher', 'people.person.profession', 'Record producer'), ('As Long As You Love Me (Audiobot remix)', 'music.recording.artist', 'Justin Bieber'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fsz2'), ('Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0v1lwt2'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Taylor Swift', 'people.person.profession', 'Actor'), ('Believe', 'music.album.artist', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'Paramount Pictures'), ('Heartbreaker', 'music.recording.song', 'Heartbreaker'), ('m.0njgyk4', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Album'), ('Teyana', 'people.person.gender', 'Female'), ('Chris Jasper', 'freebase.valuenotation.has_value', 'Parents'), ('Foreign Remix', 'common.topic.notable_for', 'g.1q67phzjh'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miley Cyrus', 'people.person.nationality', 'United States of America'), ('Shaggy', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Johnny Crawford', 'people.person.profession', 'Musician'), ('m.0njhxd_', 'award.award_honor.award', 'Billboard Music Award for Top Digital Media Artist'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.artist', 'Justin Bieber'), ('Rihanna', 'music.artist.genre', 'Rhythm and blues'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cherish'), ('Soulja Boy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Confident', 'music.recording.featured_artists', 'Chance the Rapper'), ('Christina Milian', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Date of birth'), (\"1Club.FM: Jammin' Oldies\", 'common.topic.notable_types', 'Broadcast Content'), ('PYD', 'common.topic.notable_types', 'Musical Album'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0w3gbtv'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.011j_d79'), ('English Language', 'language.human_language.main_country', 'Canada'), ('Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'), ('Lady Antebellum', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Official website', 'rdf-schema#domain', 'Topic'), ('School Boy Records', 'music.record_label.artist', 'Carly Rae Jepsen'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat'), ('m.0ywtfj6', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0sgkw_d', 'freebase.valuenotation.has_no_value', 'Winning work'), ('1Club.FM: Mix 106', 'common.topic.webpage', 'm.0430821'), ('Justin Bieber', 'music.artist.album', 'All That Matters'), ('Estelle', 'music.artist.genre', 'Electronic dance music'), ('m.0z85qxq', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0v_6zk4', 'tv.tv_guest_personal_appearance.appearance_type', 'Him/Herself'), ('Ronald Isley', 'music.composer.compositions', 'Bad Day'), ('Jaden Smith', 'people.person.gender', 'Male'), ('m.0dj34q5', 'common.webpage.category', 'About'), ('Selena Gomez', 'music.artist.genre', 'Pop music'), ('PowerHitz', 'common.topic.notable_types', 'Broadcast Content'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ndc0sf'), ('Gwen Stefani', 'people.person.nationality', 'United States of America'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Buckcherry'), ('All Bad', 'common.topic.notable_types', 'Composition'), ('The Black Eyed Peas', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Never Say Never', 'common.topic.article', 'm.0c3vvnp'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY radio)'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'music.recording.song', 'Beauty And A Beat'), ('Hold Tight', 'freebase.valuenotation.is_reviewed', 'Composer'), ('Jason Mraz', 'music.artist.genre', 'Rock music'), ('JellyRadio.com', 'broadcast.content.artist', 'Jayceon Terrell Taylor'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Audien Luvstep mix)'), ('Kelis', 'music.artist.genre', 'Dance-pop'), ('Nelly Furtado', 'music.artist.genre', 'Pop music'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Award category'), ('m.09xwqmn', 'common.webpage.topic', 'Record producer'), ('Rudolph Isley', 'people.person.nationality', 'United States of America'), ('Eenie Meenie', 'common.topic.notable_types', 'Canonical Version'), ('Die in Your Arms', 'music.composition.composer', 'Kelly Lumpkins'), ('Singing', 'people.professional_field.professions_in_this_field', 'Singer'), ('Nelly Furtado', 'music.artist.genre', 'Contemporary R&B'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Estelle', 'music.artist.genre', 'Dance-pop'), ('Under the Mistletoe', 'common.topic.notable_for', 'g.1255l1vtl'), ('m.0gbmnsc', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Sir Mix-a-Lot'), ('All Bad', 'music.composition.composer', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvzb'), ('Nelly Furtado', 'people.person.nationality', 'Canada'), ('SoulfulClassics.com', 'broadcast.content.genre', 'Pop music'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Young Hollywood Award for Newcomer of the Year', 'award.award_category.nominees', 'm.0yr9tjb'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0pbzq13', 'film.performance.character', 'Alien on TV Monitors #2'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Rise to Fame', 'common.topic.notable_types', 'Film'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njdq6k'), ('Under the Mistletoe', 'music.album.genre', 'Pop music'), ('Brandy Norwood', 'people.person.gender', 'Female'), ('Kid Cudi', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Rihanna', 'broadcast.artist.content', 'Hot Wired Radio'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Ciara'), ('Jane Lipsitz', 'people.person.profession', 'Film Producer'), ('Chingy', 'broadcast.artist.content', '.977 The Hits Channel'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('American Music Awards of 2012', 'award.award_ceremony.nominees', 'm.0n58kgb'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Matchbox Twenty'), ('Ontario', 'location.location.time_zones', 'Eastern Time Zone'), ('One Less Lonely Girl', 'music.album.releases', 'One Less Lonely Girl'), ('Ludacris', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Abriel'), ('Keyshia Cole', 'people.person.languages', 'English Language'), ('Dany Brillant', 'people.person.profession', 'Singer'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.composer.compositions', 'As Long as You Love Me'), ('Jordan Francis', 'music.artist.genre', 'Teen pop'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten club dub)'), ('Daniel Bedingfield', 'broadcast.artist.content', '1Club.FM: Power'), ('Jessica Simpson', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'music.artist.album', 'My Worlds'), ('Keyshia Cole', 'people.person.gender', 'Female'), ('Toby Gad', 'people.person.gender', 'Male'), ('Where Are Ü Now', 'common.topic.notable_for', 'g.11b7fxl7j1'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Cascada'), ('Lupe Fiasco', 'freebase.valuenotation.has_value', 'Parents'), ('Live My Life (Jaywalker remix)', 'music.recording.artist', 'Far East Movement'), ('RBMG Records', 'music.record_label.artist', 'Scooter Braun'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0yrk18w', 'award.award_honor.ceremony', '2011 Teen Choice Awards'), ('Ciara', 'music.artist.genre', 'Contemporary R&B'), ('PYD', 'music.composition.composer', 'Justin Bieber'), ('Lady Gaga', 'broadcast.artist.content', '1Club.FM: Power'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Musician', 'common.topic.subject_of', 'Stephen Melton'), ('Beauty and a Beat (Remixes)', 'music.album.album_content_type', 'Remix album'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.composition.recordings', 'Baby'), ('radioIO RNB Mix', 'broadcast.content.location', 'Tampa'), ('Lolly', 'music.album.release_type', 'Single'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0z87d3n'), ('m.0gwhmhm', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0v_714c', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Winning work', 'type.property.expected_type', 'Award-Winning Work'), ('Lil Jon', 'people.person.nationality', 'United States of America'), ('All Bad', 'common.topic.notable_for', 'g.1yl5r3f4x'), ('Area codes 519 and 226', 'location.location.containedby', 'Ontario'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjgrzc'), ('Lady Gaga', 'broadcast.artist.content', 'PowerHitz'), ('Alicia Keys', 'music.artist.genre', 'Pop music'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j2189_'), ('Baby', 'music.composition.recordings', 'Baby'), ('m.0yrkc0l', 'award.award_honor.honored_for', 'Boyfriend'), ('Britney Spears', 'broadcast.artist.content', 'radioIO Todays POP'), ('Jason Mraz', 'people.person.profession', 'Film Producer'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yr9c1k'), ('Ginuwine', 'broadcast.artist.content', '1Club.FM: V101'), ('Fergie', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Amerie', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z83x9l'), ('justinbieber', 'common.topic.notable_types', 'Award-Winning Work'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0tjyljn'), ('JoJo', 'people.person.profession', 'Singer-songwriter'), ('JoJo', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0y6dqtf', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('CL', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.01053qzf'), ('As Long as You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Author', 'freebase.type_hints.included_types', 'Person'), ('Bad Day', 'music.album.primary_release', 'Bad Day'), ('#thatPOWER', 'music.album.artist', 'Will i Am'), ('Shaffer Smith', 'broadcast.artist.content', '1Club.FM: V101'), ('Johnny Crawford', 'people.person.gender', 'Male'), ('Nelly Furtado', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Hip hop music', 'broadcast.genre.content', 'JellyRadio.com'), ('m.0y5th3r', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('P!nk', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'music.artist.genre', 'Dance music'), ('Lil Wayne', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Gender'), ('As Long As You Love Me', 'common.topic.notable_types', 'Musical Recording'), ('Carrie Underwood', 'people.person.profession', 'Singer-songwriter'), ('Mason Levy', 'freebase.valuenotation.has_value', 'Date of birth'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jet Black Stare'), ('Tyga', 'music.composer.compositions', 'Wait for a Minute'), ('m.0j8z6tl', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Mariah Carey', 'people.person.profession', 'Musician'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audien dubstep edit)'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Audien dubstep edit)'), ('m.0rqh7d9', 'tv.tv_guest_personal_appearance.episode', 'Justin Bieber'), ('Justin bieber', 'common.image.appears_in_topic_gallery', 'Justin Bieber (VEVO Channel)'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Height'), ('Enrique Iglesias', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhxzc'), ('Katy Perry: Part of Me', 'film.film.production_companies', 'MTV Films'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8755b'), ('Gas Pedal', 'common.topic.notable_for', 'g.11b5l_z7r1'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv74'), ('Never Say Never (acoustic)', 'music.recording.artist', 'Justin Bieber'), ('Never Say Never: The Remixes', 'common.topic.article', 'm.0g9y40j'), ('m.0njw4z2', 'freebase.valuenotation.has_no_value', 'Winning work'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Chingy'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Boyfriend', 'award.award_nominated_work.award_nominations', 'm.0v1d2xz'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Parents', 'type.property.expected_type', 'Person'), ('Justin Bieber: Never Say Never', 'film.film.music', 'Edvard Grieg'), ('Iggy Azalea', 'freebase.valuenotation.has_value', 'Parents'), ('justinbieber', 'common.topic.notable_types', 'Blog'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101fsyr'), ('Whitney Houston', 'broadcast.artist.content', '1Club.FM: V101'), ('Michael Jackson', 'music.artist.genre', 'Contemporary R&B'), ('Nicki Minaj', 'music.artist.contribution', 'm.0vp755b'), ('Elvis Presley', 'people.person.nationality', 'United States of America'), ('Baby', 'music.single.versions', 'Baby'), ('Under the Mistletoe', 'music.album.album_content_type', 'Studio album'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jon McLaughlin'), ('m.0yrkgd6', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('David Nicksay', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Documentary film'), ('The Roots', 'music.artist.label', 'The Island Def Jam Music Group'), ('Michael Jackson', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Cobra Starship'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beast'), ('Jordin Sparks', 'people.person.profession', 'Singer-songwriter'), ('Next to You', 'music.recording.artist', 'Chris Brown'), ('Jon M. Chu', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Selena Gomez', 'freebase.valuenotation.has_no_value', 'Children'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.song', 'As Long as You Love Me'), ('Justin Timberlake', 'broadcast.artist.content', 'FLOW 103'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Ciara'), ('Baby', 'common.topic.notable_types', 'Award-Winning Work'), ('American Music Award for Artist of the Year', 'award.award_category.winners', 'm.0njdns_'), ('Adam Messinger', 'music.lyricist.lyrics_written', 'That Should Be Me'), ('Rita Ora', 'people.person.profession', 'Actor'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Johnny Crawford', 'people.person.languages', 'English Language'), ('One Time', 'common.topic.notable_for', 'g.1256drntj'), ('m.0gbmnsk', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('2010 MTV Video Music Brazil', 'award.award_ceremony.awards_presented', 'm.0yrhhqv'), ('Madonna', 'people.person.profession', 'Film Producer'), ('Carrie Underwood', 'people.person.gender', 'Female'), ('Sean Combs', 'music.artist.genre', 'Hip hop music'), ('m.0101fszb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Kings of Leon', 'music.artist.genre', 'Indie rock'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Justin Bieber'), ('m.0z87597', 'award.award_nomination.ceremony', 'NME Awards 2012'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Selena Gomez'), ('Year', 'rdf-schema#range', 'Date/Time'), ('Demi Lovato', 'people.person.profession', 'Musician'), ('Ernie Isley', 'people.person.profession', 'Singer'), ('Trick Daddy', 'people.person.profession', 'Record producer'), ('Dance music', 'broadcast.genre.content', 'NonStopPlay.com'), ('Enrique Iglesias', 'music.artist.genre', 'Electronic dance music'), ('Britney Spears', 'people.person.profession', 'Record producer'), ('Bigger', 'music.composition.form', 'Song'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', \"Destiny's Child\"), ('Jessie J', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('1Club.FM: Power', 'broadcast.content.artist', 'Sean Paul'), ('Khalil', 'music.artist.genre', 'Rhythm and blues'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Chris Brown'), ('Lolly', 'common.topic.notable_for', 'g.11b75p5t_1'), ('Terius Nash', 'music.composer.compositions', 'Baby'), ('Gas Pedal', 'music.recording.featured_artists', 'Justin Bieber'), ('John Mamann', 'people.person.profession', 'Musician'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_4d6'), ('Yves Bole', 'music.artist.album', 'Sunday Morning'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'KooL CrAzE'), ('Die in Your Arms', 'music.recording.artist', 'Justin Bieber'), ('30 Days in May', 'film.film.personal_appearances', 'm.0y5tj13'), ('Justin Bieber', 'music.composer.compositions', 'Recovery'), ('PYD', 'music.recording.artist', 'Justin Bieber'), ('One Time', 'music.album.primary_release', 'One Time'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Lady Antebellum'), ('m.09y89l2', 'common.webpage.topic', 'Teen idol'), ('Miley Cyrus', 'base.icons.icon.icon_genre', 'Teen idol'), ('Kelly Clarkson', 'broadcast.artist.content', 'radioIO Todays POP'), ('Under the Mistletoe', 'music.album.genre', 'Christmas music'), ('.977 The Hits Channel', 'broadcast.content.artist', 'No Doubt'), ('Live My Life', 'music.recording.featured_artists', 'Justin Bieber'), ('Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'), ('Lil Jon', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (Dada Life remix)'), ('m.0vp7cl4', 'music.recording_contribution.contributor', 'Usher'), ('Justin Timberlake', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Hikaru Utada', 'music.artist.genre', 'Rock music'), ('Justin Bieber', 'music.artist.album', 'Change Me'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvhv'), ('Kanye West', 'music.artist.origin', 'Chicago'), ('m.0y4tdml', 'award.award_honor.award', 'Radio Disney Music Award for Best Male Artist'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0ndc259', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Album'), ('Justin Bieber', 'music.featured_artist.recordings', 'Lolly'), ('Ronald Isley', 'music.artist.label', 'The Island Def Jam Music Group'), ('Cory Gunz', 'music.artist.genre', 'Pop music'), ('50 Cent', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('1Club.FM: 80s (Pop)', 'broadcast.content.location', 'Chicago'), ('Gwen Stefani', 'music.artist.genre', 'Hip hop music'), ('Ray J', 'people.person.profession', 'Singer'), ('Jon M. Chu', 'people.person.profession', 'Film Producer'), ('Rihanna', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'music.artist.genre', 'Contemporary R&B'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Juvenile'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Hoobastank'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Country of nationality'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Boys Like Girls'), ('1.FM Top 40', 'broadcast.content.artist', 'Don Henley'), ('Nicki Minaj', 'people.person.profession', 'Artist'), ('Trey Songz', 'music.artist.genre', 'Hip hop music'), ('Katy Perry: Part of Me', 'film.film.language', 'English Language'), ('181-beat', 'broadcast.content.artist', 'Fabolous'), ('Fabolous', 'broadcast.artist.content', 'HitzRadio.com'), ('Believe Acoustic', 'award.award_nominated_work.award_nominations', 'm.0v30sv7'), ('Demi Lovato', 'music.artist.genre', 'Dance-pop'), ('Anastacia', 'people.person.place_of_birth', 'Chicago'), ('#thatPower', 'award.award_nominated_work.award_nominations', 'm.0y84ynj'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Anastacia'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3g2'), ('Profession', 'owl#inverseOf', 'People With This Profession'), ('Tricky Stewart', 'people.person.profession', 'Record producer'), ('Rob Thomas', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0jztshx', 'film.performance.special_performance_type', 'Him/Herself'), ('Teen idol', 'base.icons.icon_genre.icons', 'Christina Aguilera'), ('Rob Thomas', 'music.artist.origin', 'United States of America'), ('Love Me', 'common.topic.notable_types', 'Musical Album'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('First Dance', 'music.composition.composer', 'Justin Bieber'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Lady Gaga'), ('m.0sgk_cw', 'award.award_honor.honored_for', 'Baby'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.artist.album', 'Bad Day'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvk9'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (acoustic version)'), ('Miley Cyrus', 'celebrities.celebrity.celebrity_friends', 'm.0dm4cqr'), ('#thatPower', 'music.recording.tracks', '#thatPower'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_70fx'), ('Person', 'type.type.properties', 'Country of nationality'), ('m.0pc670l', 'award.award_honor.ceremony', '2011 MTV Movie Awards'), ('Next to You', 'music.recording.artist', 'Chris Brown'), ('m.0gbm3g2', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Author', 'freebase.type_profile.strict_included_types', 'Topic'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Contemporary R&B'), ('Raekwon', 'people.person.profession', 'Musician'), ('Chris Brown', 'broadcast.artist.content', 'Sunshine Radio'), ('Right Here (featuring Drake)', 'common.topic.notable_types', 'Musical Recording'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.composer', 'Justin Bieber'), ('Gavin DeGraw', 'people.person.profession', 'Artist'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0wjgqck', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Akon', 'people.person.profession', 'Record producer'), ('Journals', 'freebase.valuenotation.is_reviewed', 'Artist'), ('#thatPOWER', 'music.recording.song', '#thatPower'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Big Pun'), ('Katy Perry', 'people.person.languages', 'English Language'), ('Contemporary R&B', 'broadcast.genre.content', 'Hot 108 Jamz'), ('Smoothbeats', 'broadcast.content.genre', 'Urban contemporary'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Vanessa Hudgens'), ('Jay Cassidy', 'people.person.place_of_birth', 'Chicago'), ('m.0gfmm45', 'common.webpage.topic', 'Justin Bieber: Never Say Never'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0sgkw8v', 'freebase.valuenotation.has_no_value', 'Winning work'), ('HitzRadio.com', 'broadcast.content.genre', 'Top 40'), ('m.0y5tl39', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Aaliyah'), ('Selena Gomez', 'music.artist.genre', 'Synthpop'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnv1'), ('Justin Bieber', 'people.person.gender', 'Male'), ('Eenie Meenie', 'music.album.release_type', 'Single'), ('Yves Bole', 'people.person.profession', 'Blogger'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg2w8r'), ('Rhythm and blues', 'music.genre.subgenre', 'Contemporary R&B'), ('R. Kelly', 'people.person.languages', 'English Language'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Trey Songz', 'common.topic.notable_types', 'Musical Artist'), ('m.0101fv74', 'film.film_regional_release_date.film_release_region', 'Canada'), ('Christina Aguilera', 'people.person.profession', 'Singer'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('m.0ndc0sf', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Funkmaster Flex'), ('Timbaland', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.artist.album', 'Mistletoe'), ('Chance the Rapper', 'music.artist.origin', 'Chicago'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_98t5'), ('Ginuwine', 'music.artist.genre', 'Contemporary R&B'), ('m.0yrjvlh', 'award.award_honor.award', 'Teen Choice Award for Choice Music: Breakout Artist - Male'), ('Elvis Presley', 'base.icons.icon.icon_genre', 'Teen idol'), ('Yves Bole', 'people.person.parents', 'Marjan Raseni'), ('Live My Life (Party Rock remix)', 'common.topic.notable_for', 'g.1yg9d9djx'), ('The Black Eyed Peas', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7m'), ('Dr. Dre', 'broadcast.artist.content', 'HitzRadio.com'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft1d'), ('City/Town/Village', 'freebase.type_hints.included_types', 'Location'), ('CL', 'music.artist.genre', 'K-pop'), ('Never Say Never', 'music.recording.song', 'Never Say Never'), ('m.0njdns_', 'award.award_honor.award', 'American Music Award for Artist of the Year'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teen idol', 'base.icons.icon_genre.icons', 'Zac Efron'), ('HitzRadio.com', 'broadcast.content.artist', 'DHT'), ('Tommy Sands', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Fergie', 'music.artist.genre', 'Rhythm and blues'), ('Trey Songz', 'people.person.profession', 'Actor'), ('m.0y803nt', 'award.award_honor.award', 'Brit Award for International Breakthrough Act'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Recording'), ('Jay-Z', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Rodney Jerkins', 'people.person.profession', 'Musician'), ('Adrienne Bailon', 'music.artist.label', 'The Island Def Jam Music Group'), ('Shaggy', 'people.person.profession', 'Musician'), ('Roller Coaster', 'music.recording.song', 'Roller Coaster'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3f5'), ('Justin Bieber', 'common.topic.webpage', 'm.0bvmhvb'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.episode', 'Season Finale Part 2'), ('Jordin Sparks', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Yves Bole', 'base.mediaextended.youtube_channel.subscribers', 'm.0123p2z1'), ('#thatPower', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'All Bad'), ('1Club.FM: Power', 'broadcast.content.artist', 'Timbaland'), ('Dance music', 'broadcast.genre.content', 'BeirutNights.com Radio'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Fall Out Boy'), ('Yves Bole', 'influence.influence_node.influenced', 'Whitney Houston'), ('m.0yr9c1k', 'award.award_honor.ceremony', '2010 Young Hollywood Awards'), ('Beyoncé Knowles', 'people.person.profession', 'Singer'), ('Kylie Minogue', 'music.artist.genre', 'Contemporary R&B'), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Hot Wired Radio', 'broadcast.content.artist', 'Kelly Clarkson'), ('Keyshia Cole', 'people.person.profession', 'Musician'), ('Scottish Canadian', 'common.topic.notable_types', 'Ethnicity'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Paramore'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Bryan-Michael Cox', 'music.composer.compositions', 'Never Let You Go'), ('Smoothbeats', 'broadcast.content.genre', 'Hip hop music'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0sxhgqj'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9jwx'), ('WildFMRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0sgk_cw', 'award.award_honor.award_winner', 'Justin Bieber'), ('1Club.FM: Mix 106', 'broadcast.content.location', 'Chicago'), ('Khalil', 'music.artist.genre', 'Pop music'), ('m.0v90p2b', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0pcr2dh', 'tv.tv_guest_role.character', 'Jason McCann'), ('Weblink', 'type.property.authorities', 'm.0x_'), ('Justin Bieber', 'music.artist.album', 'Love Never Felt So Good'), ('Hip hop music', 'broadcast.genre.content', '1.FM Top 40'), ('Emphatic Radio.com!', 'common.topic.notable_types', 'Broadcast Content'), ('Fergie', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Juelz Santana', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber', 'music.artist.contribution', 'm.0vpggkk'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('That Should Be Me', 'common.topic.notable_types', 'Composition'), ('Red Hot Chili Peppers', 'broadcast.artist.content', 'radioIO Todays POP'), ('Nick Jonas', 'people.person.languages', 'English Language'), ('Live My Life', 'music.composition.form', 'Song'), ('David Nicksay', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Writer', 'people.profession.corresponding_type', 'Author'), ('As Long As You Love Me', 'music.recording.artist', 'Big Sean'), ('Turn to You (Mother’s Day Dedication)', 'music.album.releases', 'Turn to You (Mother’s Day Dedication)'), ('Live My Life (Party Rock remix)', 'music.recording.canonical_version', 'Live My Life'), ('Hikaru Utada', 'music.artist.genre', 'Hip hop music'), ('m.0wjhc6c', 'award.award_honor.award_winner', 'Justin Bieber'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Akon', 'music.artist.genre', 'Hip hop music'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Height'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber: Never Say Never', 'common.topic.article', 'm.0g9tcbl'), ('Taylor Swift', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Sean Kingston', 'music.artist.genre', 'Reggae'), ('Right Here', 'music.album.compositions', 'Right Here'), ('Scooter Braun', 'people.person.profession', 'Film Producer'), ('Dapo Torimiro', 'music.artist.genre', 'Dance music'), (\"Justin Bieber's Believe\", 'common.topic.notable_types', 'Film'), ('Runaway Love (remix)', 'music.album.artist', 'Kanye West'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Daniel Bedingfield'), ('radioIO RNB Mix', 'common.topic.image', 'RadioIO.png'), ('m.010htdc2', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Juelz Santana'), ('First Dance', 'music.composition.lyricist', 'Usher'), ('m.0z0tmyv', 'award.award_honor.award_winner', 'Justin Bieber'), ('FLOW 103', 'common.topic.notable_types', 'Broadcast Content'), ('Sean Combs', 'broadcast.artist.content', '1Club.FM: Power'), ('Ja Rule', 'music.artist.genre', 'Hip hop music'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_72js'), ('Frank Sinatra', 'people.person.profession', 'Singer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Hikaru Utada'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Keyshia Cole'), ('Record producer', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Bruce Dickinson'), ('m.0cvc8k4', 'common.webpage.topic', 'My World 2.0'), ('m.0_srv2b', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('WildFMRadio.com', 'broadcast.content.artist', 'Usher'), ('Teen idol', 'common.topic.webpage', 'm.09wnj4l'), ('As Long As You Love Me', 'music.recording.artist', 'Justin Bieber'), ('My Worlds: The Collection', 'common.topic.article', 'm.0gh8t69'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Beauty and a Beat', 'music.recording.song', 'Beauty And A Beat'), ('Emphatic Radio.com!', 'common.topic.article', 'm.03hs0f2'), ('Justin Bieber', 'base.popstra.celebrity.breakup', 'm.0gxnp26'), ('#thatPOWER', 'common.topic.notable_types', 'Musical Album'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Sarah Landman'), ('As Long As You Love Me', 'music.recording.canonical_version', 'As Long As You Love Me'), ('1Club.FM: Channel One', 'broadcast.content.broadcast', '1Club.FM: Channel One - 64kbps Stream'), ('The Black Eyed Peas', 'broadcast.artist.content', 'HitzRadio.com'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Date of birth'), ('Chance the Rapper', 'people.person.gender', 'Male'), ('m.0129j_53', 'education.education.student', 'Yves Bole'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Lady Gaga'), ('Kylie Minogue', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkw_d'), ('Alanis Morissette', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Coldplay', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Stephen Melton', 'music.artist.genre', 'Rock music'), ('Willa Ford', 'people.person.profession', 'Singer'), ('As Long as You Love Me', 'common.topic.notable_for', 'g.1257jn46q'), ('NME Award for Worst Album', 'award.award_category.nominees', 'm.0z87597'), ('m.0z85qxq', 'award.award_nomination.nominated_for', 'Die in Your Arms'), ('Cris Cab', 'people.person.place_of_birth', 'Miami'), ('Frank Sinatra', 'people.person.nationality', 'United States of America'), ('Justin Bieber: Rise to Fame', 'film.film.genre', 'Documentary film'), ('#thatPOWER', 'music.recording.tracks', '#thatpower'), ('Rock music', 'common.topic.subject_of', 'Stephen Melton'), ('Akon', 'music.artist.genre', 'Rhythm and blues'), ('Canada', 'location.country.languages_spoken', 'Spanish Language'), ('justinbieber', 'internet.blog.blogger', 'Justin Bieber'), ('Gwen Stefani', 'broadcast.artist.content', 'radioIO Todays POP'), ('Lolly', 'music.recording.tracks', 'Lolly'), ('Demi Lovato', 'people.person.profession', 'Singer-songwriter'), ('Ice Cube', 'people.person.profession', 'Film Producer'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Year'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award category'), ('My World', 'award.award_nominated_work.award_nominations', 'm.0z8t2dy'), (\"2012 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkw8v'), ('Die in Your Arms', 'common.topic.notable_types', 'Musical Recording'), ('Justin Bieber', 'music.artist.album', 'One Time'), ('Rodney Jerkins', 'people.person.profession', 'Record producer'), ('My World 2.0', 'common.topic.notable_types', 'Musical Album'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Ja Rule'), (\"Justin Bieber: Never Say Never - Director's Fan Cut\", 'film.film.genre', 'Music'), ('m.0v_6_ww', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Matchbox Twenty'), ('Justin Bieber', 'music.featured_artist.recordings', 'Next to You'), ('m.0v90skf', 'award.award_honor.award_winner', 'Justin Bieber'), ('L.A. Reid', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('New Kids on the Block', 'music.artist.genre', 'Contemporary R&B'), ('Cris Cab', 'music.artist.label', 'Island Records'), ('Live My Life (Party Rock remix)', 'music.recording.tracks', 'Live My Life (Party Rock remix)'), ('Chloe Bridges', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Sheryl Crow', 'people.person.profession', 'Record producer'), ('Sheryl Crow', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.0z0tgz6', 'award.award_nomination.ceremony', '4th Annual Shorty Awards'), ('Britney Spears', 'people.person.gender', 'Female'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'OneRepublic'), ('Gas Pedal', 'music.recording.tracks', 'Gas Pedal'), ('Lolly', 'common.topic.notable_for', 'g.1yfp1cfhq'), ('Ludacris', 'freebase.valuenotation.is_reviewed', 'Date of birth'), (\"O'Kelly Isley, Jr.\", 'people.person.profession', 'Singer-songwriter'), ('Runaway Love (remix)', 'music.recording.artist', 'Justin Bieber'), ('2011 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrktlv'), ('Rihanna', 'broadcast.artist.content', 'HitzRadio.com'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber: First Step 2 Forever', 'book.written_work.original_language', 'English Language'), ('WildFMRadio.com', 'broadcast.content.artist', 'Flo Rida'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Blackstreet', 'common.topic.notable_types', 'Musical Artist'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Will Smith'), ('m.0yrhmll', 'award.award_honor.ceremony', '2013 MTV Europe Music Awards'), ('Deborah Lurie', 'film.music_contributor.film', 'Katy Perry: Part of Me'), ('Sheryl Crow', 'broadcast.artist.content', 'radioIO Todays POP'), ('m.0d33hsj', 'celebrities.friendship.friend', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jennifer Lopez'), ('Jane Lipsitz', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Beautiful'), ('Singer', 'common.topic.subject_of', 'Alan Motley'), ('Favorite Girl', 'music.album.release_type', 'Single'), ('Stephen Melton', 'people.person.languages', 'English Language'), ('New Kids on the Block', 'music.artist.genre', 'Teen pop'), ('m.0z8t2dy', 'award.award_nomination.ceremony', 'NME Awards 2011'), ('Drake', 'people.person.profession', 'Record producer'), ('Believe', 'music.album.genre', 'Contemporary R&B'), ('m.0101fszs', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Colbie Caillat', 'people.person.languages', 'English Language'), ('m.07lkzw7', 'common.webpage.resource', 'm.0bldgxn'), ('Jayceon Terrell Taylor', 'common.topic.notable_types', 'Musical Artist'), ('m.0njdq6k', 'award.award_honor.award', 'American Music Award for Favorite Pop/Rock Male Artist'), ('Justin Bieber', 'people.person.sibling_s', 'm.0gxnnwc'), ('Nelly', 'broadcast.artist.content', 'PowerHitz'), ('Beauty and a Beast', 'music.recording.canonical_version', 'Beauty and a Beat'), ('m.0gbm3d3', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Baby', 'music.music_video.music_video_song', 'Baby'), ('Michael Jackson', 'music.artist.genre', 'Electronic music'), ('Red Hot Chili Peppers', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Christina Aguilera', 'people.person.gender', 'Female'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'The Roots'), ('Jordan Pruitt', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('radioIO RNB Mix', 'broadcast.content.producer', 'Radioio'), ('My Worlds Acoustic', 'music.album.primary_release', 'My Worlds Acoustic'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Drake', 'music.artist.genre', 'Contemporary R&B'), ('Baby', 'music.composition.form', 'Song'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 32kbps Stream'), ('m.0101ft2j', 'film.personal_film_appearance.person', 'Rodney Jerkins'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkgd6'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Rihanna'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5tl39'), ('Marvin Isley', 'people.person.profession', 'Musician'), ('Beauty and a Beat (Bisbetic Remix)', 'common.topic.notable_types', 'Musical Recording'), ('m.0yrktlv', 'award.award_honor.award_winner', 'Justin Bieber'), ('Chris Brown', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Taylor Swift', 'people.person.gender', 'Female'), ('Whitney Houston', 'influence.influence_node.influenced_by', 'Madonna'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Official website'), ('August Rigo', 'people.person.gender', 'Male'), ('Brandy Norwood', 'people.person.profession', 'Singer-songwriter'), ('Michael Jackson', 'influence.influence_node.influenced', 'Lady Gaga'), ('Right Here', 'music.album.artist', 'Justin Bieber'), ('Pearl Jam', 'music.artist.genre', 'Rock music'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Bryan-Michael Cox', 'music.producer.releases_produced', 'My World'), ('Ludacris', 'music.featured_artist.albums', 'Baby'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Terius Nash', 'music.artist.genre', 'Pop music'), ('PYD', 'common.topic.notable_for', 'g.1yp3ccjx9'), ('Chance the Rapper', 'music.featured_artist.recordings', 'Confident'), ('Nelly', 'music.artist.genre', 'Contemporary R&B'), ('HitzRadio.com', 'broadcast.content.artist', 'Puddle of Mudd'), ('Wont Stop (feat. Justin Bieber)', 'common.topic.notable_types', 'Musical Recording'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The All-American Rejects'), ('Don Henley', 'freebase.valuenotation.has_value', 'Parents'), ('Mariah Carey', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Alice Deejay'), ('Gwen Stefani', 'music.artist.genre', 'Dance-pop'), ('Ashlee Simpson', 'music.artist.genre', 'Electronic music'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('Mariah Carey', 'people.person.profession', 'Actor'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Jordan Francis'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Musical Album', 'type.type.properties', 'Album content type'), ('Max Martin', 'music.artist.genre', 'Dance-pop'), ('August Rigo', 'people.person.profession', 'Singer'), ('radioIO Todays RNB', 'common.topic.notable_types', 'Broadcast Content'), ('m.064xzb1', 'common.webpage.topic', 'The Island Def Jam Music Group'), ('Justin Bieber', 'people.person.profession', 'Actor'), ('Avery', 'people.person.gender', 'Female'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beast'), ('Urban contemporary', 'broadcast.genre.content', 'JellyRadio.com'), ('Don Henley', 'music.artist.genre', 'Rock music'), ('m.0njw1tn', 'award.award_honor.ceremony', '2010 MTV Europe Music Awards'), ('Juicy J', 'people.person.profession', 'Record producer'), ('Victoria Justice', 'music.artist.genre', 'Pop music'), ('Justin Bieber Concerts', 'common.resource.annotations', 'm.0r90dwg'), ('Maroon 5', 'common.topic.notable_types', 'Musical Artist'), ('Estelle', 'people.person.profession', 'Singer-songwriter'), ('m.0z898w6', 'award.award_honor.award', 'Teen Choice Award for Choice TV Villain'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv15'), ('m.0j21b47', 'music.music_video_performance.music_video_character', 'Singer'), ('m.0101ft5f', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('m.0vp7mrq', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Jay-Z', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love'), ('Contemporary R&B', 'base.webvideo.internet_video_genre.series', 'Biscuithands, The Animated Musical'), ('Bad Day', 'music.composition.composer', 'Rudolph Isley'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Baby', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Music & Concert Movies'), ('m.0_w3zrs', 'award.award_nomination.nominated_for', 'Hold Tight'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Rap music'), ('m.0njw59_', 'award.award_honor.award_winner', 'Justin Bieber'), ('Where Are Ü Now', 'music.recording.featured_artists', 'Diplo'), ('Date of birth', 'type.property.schema', 'Person'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Mariah Carey'), ('Justin Bieber: Never Say Never', 'film.film.cinematography', 'Reed Smoot'), ('Janet Jackson', 'people.person.gender', 'Female'), ('All Bad', 'music.album.releases', 'All Bad'), ('Electronic music', 'common.topic.notable_types', 'Musical genre'), ('Turn to You (Mother’s Day Dedication)', 'common.topic.notable_for', 'g.12blrwnvw'), ('Eenie Meenie', 'common.topic.notable_types', 'Musical Recording'), ('Nathan Lanier', 'film.music_contributor.film', \"Justin Bieber's Believe\"), ('Max Martin', 'film.person_or_entity_appearing_in_film.films', 'm.0101ft1d'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3d9'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Shaft'), ('Jessie J', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Jennifer Lopez', 'people.person.profession', 'Dancer'), ('The Isley Brothers', 'music.artist.genre', 'Contemporary R&B'), ('Nasri', 'music.composer.compositions', 'As Long as You Love Me'), ('Beauty And A Beat', 'music.composition.recordings', 'Beauty and a Beat (Steven Redant Beauty and The Club Mix)'), ('Beauty and a Beat (Wideboys Radio Mix)', 'music.recording.featured_artists', 'Nicki Minaj'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('Raekwon', 'music.artist.genre', 'Hip hop music'), ('Michael Jackson', 'people.person.profession', 'Musician'), ('William Orbit', 'music.artist.genre', 'Trance music'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0t4syfh'), ('m.0yrkc0l', 'award.award_honor.award', 'Teen Choice Award for Choice Single: Male Artist'), ('Maroon 5', 'music.artist.genre', 'Indie rock'), ('Justin Bieber', 'base.popstra.celebrity.supporter', 'm.0gxnp72'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('Teyana', 'people.person.profession', 'Singer-songwriter'), ('.977 The Hits Channel', 'broadcast.content.artist', 'M.I.A.'), ('As Long as You Love Me (album version)', 'music.recording.song', 'As Long as You Love Me'), ('Justin Bieber', 'music.featured_artist.recordings', 'Live My Life'), ('Ronald Isley', 'people.person.nationality', 'United States of America'), ('m.0pbzq13', 'film.performance.film', 'Men in Black 3'), ('Smoothbeats', 'broadcast.content.genre', 'Rap music'), ('m.0sgkrj4', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Bigger', 'music.recording.song', 'Bigger'), ('2011 MTV Movie Awards', 'award.award_ceremony.nominees', 'm.0pc67ly'), ('Chris Jasper', 'people.person.profession', 'Record producer'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njhxzc'), ('Contemporary R&B', 'broadcast.genre.content', '1Club.FM: 80s (Pop)'), ('Aaliyah', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Change Me', 'music.composition.composer', 'Andre Harris'), ('Everlast', 'people.person.profession', 'Musician'), ('Alanis Morissette', 'broadcast.artist.content', 'radioIO Todays POP'), ('Kelly Clarkson', 'broadcast.artist.content', '1.FM Top 40'), ('Believe Acoustic', 'music.album.primary_release', 'Believe Acoustic'), ('Thought Of You', 'music.recording.canonical_version', 'Thought of You'), ('Paul Anka', 'people.person.gender', 'Male'), ('Beyoncé Knowles', 'broadcast.artist.content', 'Hot Wired Radio'), ('Trick Daddy', 'people.person.gender', 'Male'), ('m.0yrk0mt', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Ciara'), ('Katy Perry', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0y80n25', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Alanis Morissette', 'people.person.nationality', 'Canada'), ('MTV Europe Music Award for Best Male', 'award.award_category.winners', 'm.0njw4z2'), ('Hot Wired Radio', 'broadcast.content.artist', 'Leona Lewis'), ('m.0sgkyfg', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('As Long As You Love Me (Audiobot remix)', 'common.topic.notable_types', 'Musical Recording'), ('Alanis Morissette', 'people.person.profession', 'Singer'), ('Geri Halliwell', 'common.topic.notable_types', 'Musical Artist'), ('Artist', 'common.topic.subject_of', 'Brian Keith Kennedy'), ('Right Here', 'music.composition.recorded_as_album', 'Right Here'), ('Book', 'freebase.type_hints.included_types', 'Written Work'), ('Rodney Jerkins', 'common.topic.notable_types', 'Record Producer'), ('Justin Bieber: First Step 2 Forever', 'book.written_work.author', 'Justin Bieber'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Believe', 'music.album.releases', 'Believe'), ('Jaden Smith', 'freebase.valuenotation.has_no_value', 'Children'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0njhxzc', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Daniel Bedingfield', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Paul Anka', 'music.artist.genre', 'Rock music'), ('Willa Ford', 'music.artist.genre', 'Dance-pop'), ('Fergie', 'music.artist.genre', 'Dance-pop'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Chris Brown'), ('1Club.FM: V101', 'broadcast.content.artist', 'Ginuwine'), ('Jaden Smith', 'people.person.parents', 'Will Smith'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_types', 'Canonical Version'), ('PYD', 'music.recording.featured_artists', 'R. Kelly'), ('m.0101ftt1', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('2010 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw257'), ('Never Let You Go', 'music.composition.composer', 'Justin Bieber'), ('Smoothbeats', 'broadcast.content.genre', 'Hip hop music'), ('Singer-songwriter', 'fictional_universe.character_occupation.characters_with_this_occupation', 'Robby Stewart'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (PAULO & JACKINSKY dub)'), ('1Club.FM: Power', 'broadcast.content.artist', 'Chris Brown'), ('August Rigo', 'music.artist.genre', 'Contemporary R&B'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Rudolph Valentino', 'people.person.nationality', 'Italy'), ('#thatPOWER', 'music.recording.tracks', 'That Power'), ('Next to You', 'common.topic.notable_types', 'Musical Album'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_vw2_d'), ('m.0y5t8gm', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kelis'), ('Hot Wired Radio', 'broadcast.content.artist', 'Hinder'), ('m.012bm5cg', 'celebrities.rivalry.rival', 'iJustine'), ('.977 The Hits Channel', 'common.topic.notable_types', 'Broadcast Content'), ('Place of birth', 'type.property.reverse_property', 'People born here'), ('Justin Bieber: Just Getting Started', 'common.topic.article', 'm.0n1s4c2'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('#thatPower', 'music.recording.artist', 'Will i Am'), ('Stratford', 'location.location.time_zones', 'Eastern Time Zone'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Black Cards'), ('J. Holiday', 'people.person.profession', 'Singer'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.artist', 'Leona Lewis'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Blackstreet'), ('m.0101ft1r', 'film.personal_film_appearance.person', 'Will i Am'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z8qx3w'), ('Justin Bieber', 'music.artist.album', 'My World'), ('Gwen Stefani', 'people.person.profession', 'Dancer'), ('Baby', 'music.composition.composer', 'Ludacris'), ('Yves Bole', 'people.person.employment_history', 'm.012nv3hv'), ('Canada', 'location.country.official_language', 'English Language'), ('Live My Life', 'music.album.primary_release', 'Live My Life'), ('Rita Ora', 'music.artist.genre', 'Rock music'), ('Mason Levy', 'people.person.profession', 'Record producer'), ('Gwen Stefani', 'broadcast.artist.content', 'HitzRadio.com'), (\"Turn to You (Mother's Day Dedication)\", 'music.composition.recordings', \"Turn to You (Mother's Day Dedication)\"), ('Hot Wired Radio', 'broadcast.content.artist', 'Britney Spears'), ('Fergie', 'broadcast.artist.content', 'Hot Wired Radio'), ('Contemporary R&B', 'broadcast.genre.content', 'FLOW 103'), ('Scooter Braun', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3fr'), ('August Rigo', 'music.artist.genre', 'Dance-pop'), ('Donna Summer', 'people.person.profession', 'Actor'), ('Aaliyah', 'music.artist.genre', 'Hip hop music'), ('Urban contemporary', 'common.topic.notable_types', 'Musical genre'), ('Rita Ora', 'freebase.valuenotation.has_no_value', 'Spouse (or domestic partner)'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('m.0101ft1d', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvbf'), ('Montell Jordan', 'people.person.profession', 'Singer'), ('m.0sgkw_d', 'award.award_honor.ceremony', \"2012 Kids' Choice Awards\"), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Groove Coverage'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long as You Love Me (acoustic version)'), ('Fall Out Boy', 'music.artist.genre', 'Pop music'), ('m.0v30sv7', 'award.award_nomination.nominated_for', 'Believe Acoustic'), ('Donna Summer', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Wait for a Minute', 'music.composition.recordings', 'Wait For a Minute'), ('m.0njhtjj', 'award.award_honor.ceremony', '2011 Billboard Music Awards'), ('Justin Timberlake', 'people.person.profession', 'Dancer'), ('m.0njvtth', 'award.award_honor.ceremony', 'Juno Awards of 2011'), ('Roller Coaster', 'music.composition.composer', 'Rodney Jerkins'), ('Shorty Award for Celebrity', 'award.award_category.winners', 'm.0y_g556'), ('Lolly', 'music.album.featured_artists', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.genre', 'Rock music'), ('m.0101ftk6', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Teyana', 'people.person.profession', 'Dancer'), ('Gwen Stefani', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Pattie Mallette', 'people.person.gender', 'Female'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('First Dance', 'music.composition.lyricist', 'Alexander Parhm, Jr.'), ('Donna Summer', 'people.person.profession', 'Singer-songwriter'), ('Alien on TV Monitors #2', 'film.film_character.portrayed_in_films', 'm.0pbzq13'), ('m.0yrhhqv', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Urban contemporary', 'broadcast.genre.content', 'PowerHitz'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3fj'), ('m.0gbcs1_', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0wjgrzc', 'award.award_honor.award', 'Teen Choice Award for Choice Music - Male Artist'), ('Judy Garland', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'music.composer.compositions', 'Lolly'), ('#Thatpower', 'music.recording.canonical_version', '#thatPOWER'), ('Ice Cube', 'music.artist.genre', 'Hip hop music'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('m.0ywvh8k', 'award.award_honor.award', 'Shorty Award for Celebrity'), ('Kid Cudi', 'people.person.nationality', 'United States of America'), ('Usher', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3b7'), ('m.0gbm3dn', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Contemporary R&B', 'broadcast.genre.content', 'GotRadio - RnB Classics'), ('m.0yr9c1k', 'award.award_honor.award', 'Young Hollywood Award for Newcomer of the Year'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World 2.0', 'music.album.releases', 'My World 2.0'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Mary J. Blige'), ('Nelly Furtado', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('m.0sxhq3d', 'award.award_nomination.nominated_for', 'Believe'), ('justinbieber', 'award.award_winning_work.awards_won', 'm.0ywvh8k'), ('Alicia Keys', 'people.person.nationality', 'United States of America'), ('My World', 'common.topic.notable_types', 'Musical Album'), ('Mannie Fresh', 'music.artist.genre', 'Hip hop music'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Chris Brown', 'broadcast.artist.content', '.977 The Hits Channel'), ('All Bad', 'music.album.artist', 'Justin Bieber'), ('Bryan-Michael Cox', 'people.person.place_of_birth', 'Miami'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Jordin Sparks', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Change Me', 'music.recording.song', 'Change Me'), ('m.0dj34q5', 'common.webpage.topic', 'Justin Bieber'), ('Boyfriend', 'music.album.compositions', 'Boyfriend'), ('50 Cent', 'people.person.profession', 'Actor'), ('Christina Milian', 'broadcast.artist.content', '1.FM Top 40'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftrz'), ('Wait for a Minute', 'common.topic.notable_for', 'g.1yl5t5h0r'), ('Christina Aguilera', 'broadcast.artist.content', '1.FM Top 40'), ('Jessica Simpson', 'broadcast.artist.content', 'radioIO Todays POP'), ('SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Shaggy'), ('Will i Am', 'music.artist.genre', 'Hip hop music'), ('John Mamann', 'people.person.profession', 'Singer'), ('Frank Sinatra', 'people.person.profession', 'Film Producer'), ('DMX', 'music.artist.genre', 'Hip hop music'), ('Mariah Carey', 'music.artist.genre', 'Hip hop music'), ('Lil Wayne', 'broadcast.artist.content', 'PowerHitz'), ('Justin Bieber', 'music.composer.compositions', 'Roller Coaster'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Justin Timberlake'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Hot Wired Radio', 'broadcast.content.artist', 'Avril Lavigne'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Akon', 'freebase.valuenotation.is_reviewed', 'Gender'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0_syttc'), ('Johntá Austin', 'music.artist.label', 'Island Records'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Gavin DeGraw'), ('Never Let You Go', 'common.topic.notable_for', 'g.1z2stmrpq'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mark Morrison'), ('#thatPOWER', 'music.album.featured_artists', 'Justin Bieber'), ('m.0z87d3n', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvtth'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Profession'), (\"O'Kelly Isley, Jr.\", 'music.artist.genre', 'Rock music'), (\"Kids' Choice Award for Favorite Male Singer\", 'award.award_category.winners', 'm.0sgkrj4'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Pop music'), ('Alanis Morissette', 'people.person.profession', 'Musician'), ('BeirutNights.com Radio', 'common.topic.notable_for', 'g.1259zkvd0'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk4gn'), ('All Around the World', 'common.topic.notable_for', 'g.12v_hb5zc'), ('m.0101ft1d', 'film.personal_film_appearance.person', 'Max Martin'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjsq'), ('Baby', 'common.topic.notable_for', 'g.125g47b9s'), ('Teen Choice Award for Choice Music - Male Artist', 'award.award_category.winners', 'm.0yrk18w'), ('First Dance', 'music.composition.lyricist', 'Jesse Wilson'), ('2013 Radio Disney Music Awards', 'award.award_ceremony.nominees', 'm.0_vmmj6'), ('Sean Kingston', 'music.artist.genre', 'Rhythm and blues'), ('Kuk Harrell', 'people.person.place_of_birth', 'Chicago'), ('m.0yrkgd6', 'award.award_honor.award_winner', 'Justin Bieber'), ('First Dance', 'freebase.valuenotation.is_reviewed', 'Lyricist'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sara Bareilles'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Jay-Z'), ('m.0n1ykxp', 'award.award_honor.award_winner', 'Justin Bieber'), ('Keyshia Cole', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft3d'), ('Yves Bole', 'music.artist.album', 'Look Inside Of Me'), ('JellyRadio.com', 'common.topic.notable_types', 'Broadcast Content'), ('One Time', 'music.album.releases', 'One Time'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Trance music'), ('Singer', 'common.topic.image', \"Ercole de' Roberti 003\"), ('Chicago', 'base.biblioness.bibs_topic.subsumes', 'Chicago'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Gender'), ('CL', 'music.artist.label', 'School Boy Records'), ('Boyfriend', 'music.recording.song', 'Boyfriend'), ('Billboard Music Award for Top Male Artist', 'award.award_category.winners', 'm.0v90skf'), ('Rihanna', 'music.artist.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.artist', 'Shaffer Smith'), ('Redfoo', 'music.featured_artist.recordings', 'Live My Life (Party Rock remix)'), ('Boyfriend', 'music.album.releases', 'Boyfriend'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Nicki Minaj', 'music.artist.genre', 'Contemporary R&B'), ('Hold Tight', 'common.topic.notable_for', 'g.1yg4dn910'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'Douglas C. Merrifield'), ('Turn to You (Mother’s Day Dedication)', 'music.album.releases', 'Turn to You (Mother’s Day Dedication)'), ('#Thatpower', 'music.recording.featured_artists', 'Justin Bieber'), ('Timbaland', 'broadcast.artist.content', 'WildFMRadio.com'), ('R. Kelly', 'people.person.profession', 'Singer-songwriter'), ('m.012m1vnr', 'music.group_membership.role', 'Record producer'), ('m.0dm4cqr', 'celebrities.friendship.friend', 'Justin Bieber'), ('Baby', 'music.album.primary_release', 'Baby'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njw4z2'), ('Donna Summer', 'music.artist.genre', 'Dance music'), ('m.0ndc259', 'award.award_honor.honored_for', 'Believe'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Height'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Twista', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Everlast', 'people.person.gender', 'Male'), ('Fabolous', 'music.artist.genre', 'Hip hop music'), ('Under the Mistletoe', 'music.album.artist', 'Justin Bieber'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Christina Aguilera', 'music.artist.genre', 'Teen pop'), ('Christina Aguilera', 'music.artist.genre', 'Hip hop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Brandy Norwood', 'music.artist.genre', 'Dance-pop'), ('Love Never Felt So Good', 'music.album.artist', 'Michael Jackson'), ('m.0njw257', 'award.award_honor.award', 'MTV Europe Music Award for Best Push Artist'), ('Country of nationality', 'rdf-schema#domain', 'Person'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Jay Cassidy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Chris Brown', 'people.person.profession', 'Actor'), ('1Club.FM: Power', 'broadcast.content.artist', 'Shaffer Smith'), ('Year', 'type.property.expected_type', 'Date/Time'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Mannie Fresh'), ('Haley James Scott', 'fictional_universe.fictional_character.occupation', 'Singer'), ('Willa Ford', 'people.person.profession', 'Dancer'), ('Tupac Shakur', 'broadcast.artist.content', '1Club.FM: Power'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njwb81'), (\"Justin Bieber's Believe\", 'film.film.produced_by', 'Stuart Ford'), (\"Justin Bieber's Believe\", 'film.film.runtime', 'm.0101ftww'), ('Rihanna', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Urban contemporary', 'broadcast.genre.content', 'Hot Wired Radio'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Frank Sinatra', 'people.person.gender', 'Male'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Club Mix)'), ('Sheryl Crow', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), (\"Bill O'Dowd\", 'freebase.valuenotation.has_value', 'Date of birth'), ('Indie rock', 'common.topic.notable_types', 'Musical genre'), ('Usher', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Ja Rule', 'people.person.gender', 'Male'), ('Tyga', 'people.person.nationality', 'United States of America'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.winners', 'm.0yrkc0l'), ('Sean Combs', 'music.artist.genre', 'Dance-pop'), ('Justin Bieber', 'base.icons.icon.icon_genre', 'Teen idol'), ('Pray', 'music.composition.recordings', 'Baby (acoustic)'), ('m.0bnstws', 'common.webpage.in_index', 'Blissful Celebrities'), ('HitzRadio.com', 'broadcast.content.artist', 'Snoop Dogg'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (Neon NiteClub Remix)'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Musical Album', 'freebase.type_profile.equivalent_topic', 'Album'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.composer.compositions', 'Beauty And A Beat'), ('m.0gxnp5m', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Dr. Dre', 'people.person.profession', 'Music executive'), ('Dance music', 'broadcast.genre.content', \"1Club.FM: Jammin' Oldies\"), ('#thatPOWER', 'common.topic.notable_for', 'g.1q3sj3yp4'), ('HitzRadio.com', 'broadcast.content.artist', 'Mis-Teeq'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3fr'), ('m.0y80n25', 'award.award_nomination.nominated_for', 'Heartbreaker'), ('m.0bnstws', 'common.webpage.topic', 'Justin Bieber'), ('Miley Cyrus', 'music.artist.genre', 'Dance-pop'), ('Katy Perry', 'music.artist.genre', 'Synthpop'), ('Kanye West', 'people.person.profession', 'Singer-songwriter'), ('Ellen DeGeneres', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvwd'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'people.person.profession', 'Artist'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvqf'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fall Out Boy', 'music.artist.genre', 'Rock music'), ('Yves Bole', 'music.artist.album', 'Raindrops'), ('Next to You', 'common.topic.notable_types', 'Musical Recording'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Profession'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Adriano Celentano'), ('Sunshine Radio', 'broadcast.content.language', 'Hungarian language'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Ciara'), ('m.0ywsnc9', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Justin Bieber', 'music.featured_artist.recordings', 'Runaway Love (remix)'), ('1.FM Top 40', 'common.topic.notable_types', 'Broadcast Content'), ('Nicki Minaj', 'music.artist.contribution', 'm.0vp8rhw'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_70fx'), ('Fergie', 'people.person.profession', 'Singer-songwriter'), ('P!nk', 'common.topic.notable_types', 'Musical Artist'), ('Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('United States of America', 'location.country.administrative_divisions', 'Chicago'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0v30srx'), ('Mannie Fresh', 'broadcast.artist.content', 'Hot 108 Jamz'), ('m.0y_g556', 'award.award_honor.award_winner', 'Justin Bieber'), ('radioIO Classic RNB', 'broadcast.content.producer', 'Radioio'), ('m.0j219nq', 'music.music_video_performance.music_video_character', 'Singer'), ('Billboard Music Award for Top Digital Media Artist', 'award.award_category.winners', 'm.0njhxd_'), ('Christina Aguilera', 'base.icons.icon.icon_genre', 'Teen idol'), ('Daniel Bedingfield', 'music.artist.label', 'Island Records'), ('Justin Bieber', 'film.actor.film', 'm.0pbzq13'), ('Marvin Isley', 'music.artist.genre', 'Rhythm and blues'), ('m.012m1vf1', 'music.group_membership.role', 'Record producer'), ('Singer', 'theater.theater_character.portrayed_by', 'm.012nh7h_'), ('As Long as You Love Me', 'common.topic.notable_types', 'Composition'), ('Britney Spears', 'people.person.nationality', 'United States of America'), ('Christina Milian', 'people.person.gender', 'Female'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_98t5'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Gwen Stefani'), ('Ciara', 'music.artist.genre', 'Hip hop music'), ('m.0vp755b', 'music.recording_contribution.album', 'Beauty and a Beat'), ('Brandy Norwood', 'people.person.profession', 'Model'), ('Robby Stewart', 'fictional_universe.fictional_character.gender', 'Male'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cris Cab'), ('Daniel Bedingfield', 'music.artist.genre', 'Electronic dance music'), ('Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0101fvjm', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('m.0gxnnzy', 'celebrities.romantic_relationship.celebrity', 'Selena Gomez'), ('WildFMRadio.com', 'broadcast.content.genre', 'Pop music'), ('Live My Life (Party Rock remix)', 'music.recording.artist', 'Far East Movement'), ('Little Bird', 'music.recording.featured_artists', 'Redfoo'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Clay Aiken'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zbg2kw'), ('Rihanna', 'freebase.notability_hints.notable_for', 'Musical Artist'), ('Dance music', 'music.genre.subgenre', 'Pop music'), ('Jessica Simpson', 'people.person.profession', 'Actor'), ('Barry Weiss', 'business.board_member.leader_of', 'm.0ncpx3v'), ('Juelz Santana', 'people.person.profession', 'Singer'), ('Beyoncé Knowles', 'people.person.languages', 'English Language'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Tupac Shakur', 'people.person.profession', 'Dancer'), ('Twista', 'broadcast.artist.content', '1Club.FM: Power'), ('All That Matters', 'common.topic.notable_types', 'Canonical Version'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0zg8bc1', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Madonna', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Fabian', 'people.person.nationality', 'United States of America'), ('1Club.FM: Power', 'broadcast.content.artist', 'Leona Lewis'), ('m.0101fvph', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Justin Bieber'), ('Stephen Melton', 'people.person.gender', 'Male'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5d'), ('NME Award for Worst Album', 'award.award_category.winners', 'm.0z8755b'), ('Keyshia Cole', 'people.person.profession', 'Artist'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Mix 106', 'common.topic.article', 'm.04307gx'), ('Akon', 'people.person.profession', 'Actor'), ('4th Annual Shorty Awards', 'award.award_ceremony.awards_presented', 'm.0z0tmyv'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Trick Daddy'), ('My Worlds: The Collection', 'common.topic.notable_types', 'Musical Album'), ('Indie rock', 'common.topic.subject_of', 'Indie rock'), ('Fabolous', 'people.person.profession', 'Record producer'), ('Enrique Iglesias', 'people.person.languages', 'English Language'), ('Britney Spears', 'broadcast.artist.content', 'Hot Wired Radio'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0z8qqh5'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c93b'), ('Lil Jon', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Justin Bieber', 'music.featured_artist.recordings', 'Foreign Remix'), ('m.0gbm3f5', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Bigger', 'music.composition.lyricist', 'Kevin Risto'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.genre', 'Classic hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Parachute'), ('Demi Lovato', 'people.person.languages', 'English Language'), ('Gavin DeGraw', 'people.person.gender', 'Male'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Singer'), ('Boyfriend', 'music.recording.canonical_version', 'Boyfriend'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Lil Jon'), ('Vanessa Hudgens', 'people.person.profession', 'Actor'), ('Katy Perry: Part of Me', 'film.film.personal_appearances', 'm.0p85jpp'), ('Jayceon Terrell Taylor', 'broadcast.artist.content', 'JellyRadio.com'), ('HitzRadio.com', 'broadcast.content.artist', 'Ja Rule'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'September'), ('2011 Brit Awards', 'award.award_ceremony.awards_presented', 'm.0y803nt'), ('m.0bnstws', 'common.webpage.category', 'Curated Topic'), ('Fergie', 'broadcast.artist.content', 'Sunshine Radio'), ('As Long As You Love Me (Audiobot instrumental)', 'music.recording.featured_artists', 'Big Sean'), ('Bryan-Michael Cox', 'common.topic.notable_types', 'Record Producer'), ('Les Coulisses des Golden Globes', 'film.film.personal_appearances', 'm.0y5tl39'), ('Bad Day', 'music.composition.composer', \"O'Kelly Isley, Jr.\"), ('Justin Bieber', 'music.composer.compositions', 'Hold Tight'), ('Kelis', 'people.person.gender', 'Female'), ('Jon M. Chu', 'film.director.film', 'Justin Bieber: Never Say Never'), ('Dapo Torimiro', 'music.lyricist.lyrics_written', 'Bigger'), ('Bad 25', 'film.film.genre', 'Music'), ('m.0y5th3r', 'film.personal_film_appearance.film', 'Real Change: Artists for Education'), ('Christina Aguilera', 'people.person.nationality', 'United States of America'), ('Pras', 'broadcast.artist.content', 'HitzRadio.com'), ('Shaffer Smith', 'music.artist.genre', 'Hip hop music'), ('Katy Perry', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('The Pussycat Dolls', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0v90p2b', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Spouse (or domestic partner)'), ('Jessie J', 'people.person.profession', 'Artist'), ('Keyshia Cole', 'people.person.profession', 'Singer-songwriter'), ('m.0cq9hwb', 'base.popstra.friendship.participant', 'Jordan Pruitt'), ('Urban contemporary', 'music.genre.subgenre', 'Quiet Storm'), ('John Mamann', 'people.person.gender', 'Male'), ('m.0p85jpp', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Anastacia', 'music.artist.genre', 'Disco'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Hinder'), ('Alanis Morissette', 'common.topic.notable_types', 'Musical Artist'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Year'), ('1Club.FM: Power', 'broadcast.content.artist', 'Outkast'), ('Contemporary R&B', 'broadcast.genre.content', 'NonStopPlay.com'), ('The Black Eyed Peas', 'music.artist.genre', 'Dance-pop'), ('The Island Def Jam Music Group', 'organization.organization.previous_names', 'm.05sp3_n'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c927'), ('1Club.FM: Channel One', 'common.topic.article', 'm.04307hl'), ('LeAnn Rimes', 'music.artist.genre', 'Dance music'), ('Willa Ford', 'music.artist.genre', 'Pop music'), ('Runaway Love', 'common.topic.notable_types', 'Musical Recording'), ('Nathan Lanier', 'common.topic.notable_types', 'Musical Artist'), ('m.0y_g556', 'award.award_honor.award', 'Shorty Award for Celebrity'), ('Marvin Isley', 'people.person.gender', 'Male'), ('m.0101fsyr', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('Lolly', 'music.recording.song', 'Lolly'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'L.A. Reid'), ('Maroon 5', 'broadcast.artist.content', '1.FM Top 40'), ('Never Say Never: The Remixes', 'music.album.primary_release', 'Never Say Never: The Remixes'), ('First Dance', 'music.composition.composer', 'Ryan Lovette'), ('1.FM Top 40', 'broadcast.content.artist', 'Justin Bieber'), ('Justin Timberlake', 'people.person.profession', 'Musician'), ('Donna Summer', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnsy'), ('m.0v30srx', 'award.award_nomination.nominated_for', 'Believe'), ('m.0_x3kvk', 'award.award_nomination.award_nominee', 'R. Kelly'), ('Hip hop music', 'common.topic.notable_types', 'Musical genre'), ('Spouse', 'type.property.reverse_property', 'Spouse (or domestic partner)'), ('#thatPOWER', 'common.topic.notable_for', 'g.12h2w5tf4'), ('The Notorious B.I.G.', 'common.topic.notable_types', 'Musical Artist'), ('Juno Awards of 2011', 'award.award_ceremony.awards_presented', 'm.0gwhmhm'), ('Alicia Keys', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Leonardo DiCaprio', 'base.icons.icon.icon_genre', 'Teen idol'), ('Teen Choice Award for Choice Male Hottie', 'award.award_category.winners', 'm.0yrktlv'), ('Justin Bieber', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsj'), ('Fall Out Boy', 'music.artist.label', 'The Island Def Jam Music Group'), ('Fabolous', 'people.person.gender', 'Male'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Boys Like Girls'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Katy Perry'), ('Katy Perry: Part of Me', 'film.film.directed_by', 'Jane Lipsitz'), ('Blogger', 'people.profession.people_with_this_profession', 'Yves Bole'), ('m.0101ft56', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Teyana', 'people.person.nationality', 'United States of America'), ('Rita Ora', 'people.person.profession', 'Singer-songwriter'), ('All Around the World (acoustic version)', 'music.recording.song', 'All Around The World'), ('Trick Daddy', 'people.person.profession', 'Actor'), ('m.0z8qx3w', 'award.award_honor.award_winner', 'Justin Bieber'), ('Frank Ocean', 'common.topic.notable_types', 'Musical Artist'), ('m.0yrhrwc', 'award.award_honor.award', 'MTV Video Music Award Japan for Best New Artist'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv9w'), ('R. Kelly', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Alanis Morissette', 'people.person.gender', 'Female'), ('PYD', 'music.recording.song', 'PYD'), ('Somebody to Love', 'music.recording.canonical_version', 'Somebody To Love'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Urban contemporary'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Ronald Isley', 'people.person.profession', 'Musician'), ('Lupe Fiasco', 'people.person.place_of_birth', 'Chicago'), ('My World 2.0', 'music.album.genre', 'Dance-pop'), ('JoJo', 'people.person.profession', 'Singer'), ('Next to You', 'common.topic.notable_for', 'g.1yg9k0xh7'), ('Shaffer Smith', 'broadcast.artist.content', 'HitzRadio.com'), ('Chris Brown', 'broadcast.artist.content', 'Hot Wired Radio'), ('Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', 'common.topic.notable_types', 'Musical Recording'), ('Terius Nash', 'music.featured_artist.recordings', 'Baby'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Tupac Shakur'), ('Jeremy Bieber', 'common.topic.notable_for', 'g.1258wnf9x'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c91k'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Sean Combs'), ('m.0tkqqgg', 'award.award_nomination.nominated_for', 'My World'), ('Cory Gunz', 'people.person.nationality', 'United States of America'), ('1Club.FM: Power', 'broadcast.content.artist', 'Britney Spears'), ('United States of America', 'location.country.currency_used', 'United States Dollar'), ('Men in Black 3', 'film.film.language', 'English Language'), ('Nelly', 'broadcast.artist.content', '.977 The Hits Channel'), ('Linkin Park', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Justin Bieber: Just Getting Started', 'common.topic.notable_types', 'Book'), ('Ludacris', 'music.artist.genre', 'Hip hop music'), ('m.0sgkw8v', 'award.award_honor.ceremony', \"2012 Kids' Choice Awards\"), ('DMX', 'broadcast.artist.content', 'JellyRadio.com'), ('Recovery', 'music.composition.composer', 'Mark Hill'), ('Justin Bieber', 'music.featured_artist.recordings', 'Little Bird'), ('American Music Award for Artist of the Year', 'award.award_category.winners', 'm.0ndc3_1'), ('Leif Garrett', 'people.person.gender', 'Male'), ('Linkin Park', 'music.artist.genre', 'Hip hop music'), ('m.0z1jn32', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.song', 'Beauty And A Beat'), ('Ronald Isley', 'music.artist.genre', 'Contemporary R&B'), ('#thatPower', 'common.topic.notable_types', 'Musical Recording'), ('Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Jessica Simpson'), ('Savan Kotecha', 'people.person.gender', 'Male'), ('m.0njwqrb', 'award.award_honor.award_winner', 'Justin Bieber'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Tricky Stewart', 'music.composer.compositions', 'Baby'), ('The Pussycat Dolls', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Jeremy Bieber', 'people.person.children', 'Justin Bieber'), ('Justin Bieber: Never Say Never', 'film.film.directed_by', 'Jon M. Chu'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Gender'), ('Max Martin', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'common.topic.image', 'Justin bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Avril Lavigne'), ('Eenie Meenie', 'music.recording.artist', 'Sean Kingston'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2d0'), ('Journals', 'music.album.releases', 'Journals'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Rihanna'), ('Timbaland', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronski Speed', 'music.artist.genre', 'Trance music'), ('Will Smith', 'broadcast.artist.content', 'radioIO Todays POP'), ('Alicia Keys', 'broadcast.artist.content', 'radioIO RNB Mix'), (\"Turn to You (Mother's Day Dedication)\", 'music.recording.song', \"Turn to You (Mother's Day Dedication)\"), ('Nelly Furtado', 'music.artist.genre', 'Rhythm and blues'), ('New Kids on the Block', 'music.artist.genre', 'Pop music'), ('Right Here', 'music.recording.featured_artists', 'Drake'), ('Aaliyah', 'people.person.gender', 'Female'), ('m.05sp41b', 'business.company_name_change.company', 'The Island Def Jam Music Group'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Kanye West'), ('One Less Lonely Girl', 'music.album.release_type', 'Single'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c933'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9h3k'), ('Twista', 'music.artist.genre', 'Rhythm and blues'), ('m.0_grm8q', 'tv.tv_guest_personal_appearance.appearance_type', 'Celebrity guest'), (\"Justin Bieber's Believe\", 'film.film.distributors', 'm.012f2m86'), ('Driving under the influence', 'celebrities.reason_for_arrest.celebrities_charged_or_arrested', 'm.0_cyzs_'), ('Vanessa Hudgens', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Juvenile', 'freebase.valuenotation.has_value', 'Parents'), ('Snoop Dogg', 'broadcast.artist.content', 'FLOW 103'), ('Confident', 'common.topic.notable_for', 'g.1yp3dc66c'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Clay Aiken', 'people.person.nationality', 'United States of America'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Blackstreet'), ('Miley Cyrus', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Sia Furler', 'broadcast.artist.content', '1.FM Top 40'), ('m.0101gx29', 'people.marriage.spouse', 'Pattie Mallette'), ('Confident', 'music.recording.song', 'Confident'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Barcera'), ('Christina Aguilera', 'music.artist.genre', 'Pop music'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Pearl Jam', 'freebase.valuenotation.is_reviewed', 'Official website'), ('1.FM Top 40', 'broadcast.content.artist', 'Gym Class Heroes'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO Todays RNB'), ('LeAnn Rimes', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Thought of You', 'music.recording.song', 'Thought Of You'), ('Recovery', 'common.topic.notable_types', 'Composition'), ('Big R Radio - Top 40 Hits', 'common.topic.notable_for', 'g.125b5lghf'), ('m.0z83x9l', 'award.award_honor.award', 'NME Award for Villain of the Year'), ('Twista', 'music.artist.genre', 'Hip hop music'), ('m.01053qzf', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k02'), ('Ludacris', 'people.person.gender', 'Male'), ('Record producer', 'music.performance_role.guest_performances', 'm.0h3_p71'), ('Dr. Dre', 'music.artist.genre', 'Hip hop music'), ('Vocals', 'music.instrument.instrumentalists', 'Stephen Melton'), ('First Dance', 'music.composition.lyricist', 'Ryan Lovette'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftnq'), ('My Worlds Acoustic', 'common.topic.article', 'm.0fph3j0'), (\"Destiny's Child\", 'music.artist.genre', 'Teen pop'), ('Juno Fan Choice Award', 'award.award_category.winners', 'm.0njvvj_'), ('The Notorious B.I.G.', 'people.person.nationality', 'United States of America'), ('KooL CrAzE', 'music.artist.genre', 'Pop music'), ('Akon', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Anastacia', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0yrhmll', 'award.award_honor.award', 'MTV Europe Music Award for Best North American Act'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Miley Cyrus'), ('Hip hop music', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Yves Bole', 'people.person.languages', 'Spanish Language'), ('m.0j593k1', 'film.film_regional_release_date.film', 'Justin Bieber: Never Say Never'), ('Canada', 'location.location.time_zones', 'Eastern Time Zone'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Katy Perry'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c91t'), ('PowerHitz', 'broadcast.content.artist', '50 Cent'), ('Zac Efron', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Shaggy', 'broadcast.artist.content', '1Club.FM: Power'), ('Ernie Isley', 'music.artist.genre', 'Rock music'), ('Bryan-Michael Cox', 'people.person.profession', 'Record producer'), ('Amerie', 'people.person.nationality', 'United States of America'), ('m.0yqflrk', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0nfnx_3'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Will Smith', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Victoria Justice', 'music.artist.genre', 'Synthpop'), ('Justin Bieber', 'common.topic.article', 'm.06w2sn9'), ('My World', 'common.topic.image', 'My World'), ('Urban contemporary', 'broadcast.genre.content', '181-beat'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Year'), ('Shaggy', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Willa Ford', 'music.artist.origin', 'Tampa'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Rihanna'), ('Madonna', 'broadcast.artist.content', 'radioIO Todays POP'), ('RedOne', 'people.person.profession', 'Music executive'), ('Enrique Iglesias', 'broadcast.artist.content', 'radioIO Todays POP'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber Lyrics', 'common.resource.annotations', 'm.0d_hbgr'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Sisqó'), ('Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me (Ferry Corsten remix)'), ('Jon M. Chu', 'people.person.gender', 'Male'), ('m.0njvs9s', 'award.award_honor.honored_for', 'That Should Be Me'), ('As Long As You Love Me (PAULO & JACKINSKY radio)', 'music.recording.featured_artists', 'Big Sean'), ('Sir Mix-a-Lot', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Believe Acoustic', 'music.album.album_content_type', 'Remix album'), ('This is Justin Bieber', 'common.topic.notable_types', 'Film'), ('Gwen Stefani', 'broadcast.artist.content', '.977 The Hits Channel'), ('Kelis', 'music.artist.genre', 'Electronic music'), ('Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Evanescence'), ('R. Kelly', 'people.person.gender', 'Male'), ('m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Year'), ('Madonna', 'music.artist.genre', 'Dance music'), ('Country', 'freebase.type_hints.included_types', 'Location'), ('As Long as You Love Me', 'music.album.primary_release', 'As Long as You Love Me'), ('Lady Gaga', 'music.artist.genre', 'Electronic music'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Rune Reilly Kølsch'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Don Henley', 'broadcast.artist.content', '1.FM Top 40'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0vb6hhj', 'award.award_honor.award', 'Billboard Music Milestone Award'), ('#thatPOWER', 'music.recording.tracks', 'ThatPower'), ('Hold Tight', 'music.recording.song', 'Hold Tight'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('Redfoo', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Eenie Meenie', 'music.recording.artist', 'Justin Bieber'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Bad Day', 'music.recording.artist', 'Justin Bieber'), ('LeAnn Rimes', 'music.artist.genre', 'Rhythm and blues'), ('Wait For a Minute', 'music.album.primary_release', 'Wait For a Minute'), ('William Orbit', 'music.artist.genre', 'Electronic music'), ('Ontario', 'location.location.containedby', 'Canada'), ('m.012bm2d0', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Record producer', 'common.topic.webpage', 'm.09wyfdf'), ('My Worlds: The Collection', 'music.album.releases', 'My Worlds: The Collection'), ('Jessica Simpson', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ashlee Simpson', 'music.artist.genre', 'Dance-pop'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Lupe Fiasco'), ('m.0njw444', 'award.award_honor.award', 'MTV Europe Music Award for Best Male'), ('Christina Milian', 'music.artist.genre', 'Hip hop music'), ('Heartbreaker', 'music.album.release_type', 'Single'), ('Boyfriend', 'music.single.versions', 'Boyfriend'), ('Kelly Clarkson', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Beauty and a Beat (Remixes)', 'music.album.contributor', 'm.0vp8rhw'), ('m.0tjyljn', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Elvis Presley', 'music.artist.genre', 'Rock music'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Contemporary R&B'), ('Frank Ocean', 'music.artist.label', 'The Island Def Jam Music Group'), ('Katy Perry: Part of Me', 'film.film.music', 'Deborah Lurie'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Montell Jordan', 'broadcast.artist.content', 'radioIO Todays POP'), ('Chris Jasper', 'people.person.profession', 'Musician'), ('Green Day', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jazmyn Bieber', 'people.person.sibling_s', 'm.0gxnnwc'), ('Pattie Mallette', 'film.person_or_entity_appearing_in_film.films', 'm.0gz0fr6'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_6_ww'), ('American Music Awards of 2012', 'award.award_ceremony.awards_presented', 'm.0ndc0sf'), ('Ray J', 'people.person.profession', 'Actor'), ('Rhythm and blues', 'common.topic.notable_types', 'Musical genre'), ('Baby (acoustic)', 'music.recording.canonical_version', 'Pray'), ('The Isley Brothers', 'common.topic.notable_types', 'Musical Artist'), ('All Around The World', 'music.composition.composer', 'Ludacris'), ('m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('HitzRadio.com', 'broadcast.content.artist', 'Kelly Clarkson'), ('Anastacia', 'broadcast.artist.content', '1.FM Top 40'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_3bt'), ('m.0t_l7mp', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Estelle', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Adrienne Bailon'), ('Eenie Meenie', 'music.recording.featured_artists', 'Sean Kingston'), ('Hip hop music', 'broadcast.genre.content', 'PowerHitz'), ('Verse Simmonds', 'people.person.gender', 'Male'), ('m.0gxnny8', 'people.place_lived.location', 'Canada'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Linkin Park'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1.FM Top 40', 'broadcast.content.genre', 'Pop music'), ('2013 Billboard Music Awards', 'time.event.people_involved', 'Shaffer Smith'), ('Sean Kingston', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Live My Life', 'common.topic.notable_for', 'g.126tdsljg'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grm8q'), ('Juicy J', 'music.featured_artist.recordings', 'Lolly'), ('Jayceon Terrell Taylor', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Rihanna', 'broadcast.artist.content', 'PowerHitz'), ('Juvenile', 'people.person.profession', 'Actor'), ('Love Never Felt So Good', 'music.composition.language', 'English Language'), ('m.0h3_p71', 'music.recording_contribution.performance_role', 'Record producer'), ('Beauty and a Beat (Steven Redant Beauty and The Dub Mix)', 'music.recording.song', 'Beauty And A Beat'), ('RedOne', 'common.topic.notable_types', 'Record Producer'), ('Rudolph Valentino', 'people.person.profession', 'Actor'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Lolly', 'music.recording.featured_artists', 'Juicy J'), ('m.0gxnp5x', 'base.popstra.hangout.customer', 'Justin Bieber'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sean Kingston', 'music.featured_artist.recordings', 'Eenie Meenie'), ('Janet Jackson', 'music.artist.genre', 'Dance music'), ('Somebody To Love', 'music.recording.song', 'Somebody to Love'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_729v'), ('The Island Def Jam Music Group', 'organization.organization.child', 'm.0yqsnwz'), ('Pray', 'music.album.releases', 'Pray'), ('1Club.FM: Power', 'broadcast.content.artist', 'K-Ci & JoJo'), ('Lil Wayne', 'music.artist.genre', 'Contemporary R&B'), ('Whitney Houston', 'common.topic.notable_types', 'Musical Artist'), (\"Destiny's Child\", 'broadcast.artist.content', '1Club.FM: Channel One'), ('Taylor Swift', 'people.person.profession', 'Musician'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Nelly Furtado', 'people.person.profession', 'Musician'), ('1Club.FM: Power', 'broadcast.content.artist', 'Cherish'), ('1Club.FM: Power', 'broadcast.content.artist', 'Katy Perry'), ('Coldplay', 'music.artist.genre', 'Pop music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Maroon 5'), ('Blackstreet', 'music.artist.genre', 'Rhythm and blues'), ('Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Martin Kierszenbaum', 'freebase.valuenotation.has_value', 'Date of birth'), ('R. Kelly', 'music.artist.genre', 'Pop music'), ('Lupe Fiasco', 'people.person.profession', 'Record producer'), ('Bad Day', 'music.composition.composer', 'Dominic Jordan'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'OneRepublic'), ('Kuk Harrell', 'people.person.gender', 'Male'), ('Donna Summer', 'music.artist.genre', 'Pop music'), ('GotRadio - RnB Classics', 'common.topic.notable_types', 'Broadcast Content'), ('Kanye West', 'broadcast.artist.content', 'HitzRadio.com'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Flo Rida'), ('Teen idol', 'base.icons.icon_genre.icons', 'Johnny Crawford'), ('Beauty and a Beat (Remixes)', 'music.album.release_type', 'EP'), ('m.0ng_j6d', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Mistletoe', 'common.topic.notable_types', 'Musical Album'), ('Pray', 'music.recording.song', 'Pray'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'broadcast.artist.content', '1Club.FM: Power'), ('Acoustic music', 'common.topic.notable_types', 'Musical genre'), ('m.0yrkgd6', 'award.award_honor.award', 'Teen Choice Award for Choice Summer Music Star: Male'), ('Johntá Austin', 'music.artist.genre', 'Contemporary R&B'), ('Sean Combs', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Ricky Nelson', 'music.artist.genre', 'Rock music'), ('HitzRadio.com', 'broadcast.content.artist', 'Justin Bieber'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO Todays POP'), ('My Worlds', 'common.topic.notable_for', 'g.1259t8kb3'), ('Justin Timberlake', 'music.artist.genre', 'Dance music'), ('Aaliyah', 'music.artist.genre', 'Pop music'), ('Kelly Clarkson', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.09wyfdf', 'common.webpage.topic', 'Record producer'), ('Right Here', 'music.album.album_content_type', 'Studio album'), ('Chingy', 'freebase.valuenotation.has_value', 'Parents'), ('Juicy J', 'music.featured_artist.albums', 'Lolly'), ('Composition', 'type.type.properties', 'Composer'), ('#thatPower', 'music.composition.recorded_as_album', '#thatPOWER'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Journals', 'music.album.genre', 'Contemporary R&B'), ('Believe', 'music.album.releases', 'Believe'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0101fvkw', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Johntá Austin', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sia Furler', 'people.person.profession', 'Record producer'), ('Justin Bieber: Never Say Never', 'film.film.personal_appearances', 'm.0gbm3c3'), ('Mary J. Blige', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk18w'), ('Eenie Meenie', 'music.recording.song', 'Eenie Meenie'), ('Leif Garrett', 'base.icons.icon.icon_genre', 'Teen idol'), ('Profession', 'type.property.reverse_property', 'People With This Profession'), ('Terius Nash', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Aaliyah', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrk4gn'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO Classic RNB'), ('Alicia Keys', 'music.artist.genre', 'Contemporary R&B'), ('Men in Black 3', 'film.film.country', 'United States of America'), ('Whitney Houston', 'people.person.profession', 'Film Producer'), ('The Pussycat Dolls', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'common.topic.webpage', 'm.03hs4ny'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Rihanna'), ('PYD', 'common.topic.notable_for', 'g.1yw57ynz4'), ('Daniel Bedingfield', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Green Day', 'music.artist.genre', 'Rock music'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Dance music'), ('R. Kelly', 'broadcast.artist.content', '.977 The Hits Channel'), ('Jaden Smith', 'freebase.valuenotation.is_reviewed', 'Official website'), ('.977 The Hits Channel', 'broadcast.content.artist', 'TLC'), ('m.0101fvf2', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njdns_'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0sgkrj4'), ('Hot Wired Radio', 'broadcast.content.artist', \"Plain White T's\"), ('Jennifer Lopez', 'people.person.profession', 'Singer'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Boyfriend', 'music.recording.releases', 'Believe'), ('Boyfriend', 'award.award_nominated_work.award_nominations', 'm.0z84kwx'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Whitney Houston', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('As Long as You Love Me (album version)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Tricky Stewart', 'music.artist.genre', 'Pop music'), ('Caitlin Beadles', 'people.person.place_of_birth', 'Canada'), ('Fergie', 'music.artist.genre', 'Hip hop music'), ('Will Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0_cz5l3', 'celebrities.legal_entanglement.offense', 'Driving under the influence'), ('radioIO Todays POP', 'broadcast.content.broadcast', 'radioIO Todays POP - 128kbps Stream'), ('Ronald Isley', 'music.artist.genre', 'Rock music'), ('Tupac Shakur', 'broadcast.artist.content', 'JellyRadio.com'), ('Judy Garland', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Kanye West', 'music.artist.genre', 'Hip hop music'), ('Christina Aguilera', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Justin Bieber', 'music.composer.compositions', \"Turn to You (Mother's Day Dedication)\"), ('Savan Kotecha', 'music.artist.genre', 'Pop music'), ('Fabian', 'people.person.profession', 'Singer'), ('Hikaru Utada', 'common.topic.notable_types', 'Musical Artist'), ('Jay-Z', 'music.artist.genre', 'Hip hop music'), ('The Island Def Jam Music Group', 'award.award_nominee.award_nominations', 'm.0ws63cr'), ('Terius Nash', 'broadcast.artist.content', 'HitzRadio.com'), ('Rob Thomas', 'music.artist.genre', 'Pop music'), ('Tampa', 'location.location.time_zones', 'Eastern Time Zone'), ('Live My Life (Jaywalker remix)', 'music.recording.song', 'Live My Life'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Ferry Corsten radio)'), ('Hit-Boy', 'people.person.gender', 'Male'), ('Justin Bieber', 'internet.blogger.blog', 'justinbieber'), ('Zendaya: Behind the Scenes', 'film.film.language', 'English Language'), ('1Club.FM: V101', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long as You Love Me', 'music.composition.recorded_as_album', 'As Long as You Love Me'), ('Taylor Swift', 'celebrities.celebrity.celebrity_friends', 'm.0_grqb8'), ('Hot Wired Radio', 'broadcast.content.artist', 'Carrie Underwood'), ('m.0z85qxq', 'award.award_nomination.ceremony', '2012 Teen Choice Awards'), ('Savan Kotecha', 'music.composer.compositions', 'Beauty And A Beat'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Justin Timberlake'), ('Jennifer Lopez', 'broadcast.artist.content', 'radioIO Todays POP'), ('John Mamann', 'music.artist.genre', 'Pop music'), ('Teen idol', 'base.icons.icon_genre.icons', 'Rudolph Valentino'), ('m.0jsmvv5', 'film.film_regional_release_date.film_release_region', 'United States of America'), ('HitzRadio.com', 'broadcast.content.artist', 'Lil Rob'), ('Hot Wired Radio', 'broadcast.content.artist', 'The All-American Rejects'), ('m.0sgk_cw', 'award.award_honor.ceremony', \"2011 Kids' Choice Awards\"), ('Ronald Isley', 'music.artist.genre', 'Hip hop music'), ('Official website', 'type.property.expected_type', 'URI'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.song', 'As Long as You Love Me'), ('J. Holiday', 'music.artist.genre', 'Hip hop music'), ('Kings of Leon', 'common.topic.notable_types', 'Musical Artist'), ('My World', 'common.topic.notable_for', 'g.125g8b_3v'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj_1'), ('Pras', 'people.person.gender', 'Male'), ('Ashley Tisdale', 'base.icons.icon.icon_genre', 'Teen idol'), ('Singer-songwriter', 'base.musicpf.genre.artist', 'Yves Bole'), ('m.0gxnnwc', 'people.sibling_relationship.sibling', 'Justin Bieber'), ('m.0_grqb8', 'celebrities.friendship.friend', 'Justin Bieber'), ('P!nk', 'broadcast.artist.content', 'radioIO Todays POP'), ('Shorty Award for Celebrity', 'award.award_category.winners', 'm.0ywvh8k'), ('HitzRadio.com', 'broadcast.content.genre', 'Urban contemporary'), ('Lolly', 'common.topic.notable_for', 'g.12v_h9661'), ('Live My Life (Jaywalker remix)', 'common.topic.notable_for', 'g.1yg9hnqh7'), ('Rihanna', 'music.artist.genre', 'Eurodance'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Height'), ('Iggy Azalea', 'music.artist.genre', 'Hip hop music'), ('The Roots', 'music.artist.genre', 'Rock music'), ('Nathan Lanier', 'people.person.profession', 'Musician'), ('Ginuwine', 'music.artist.origin', 'United States of America'), ('Beyoncé Knowles', 'people.person.profession', 'Musician'), ('Bigger', 'common.topic.notable_types', 'Canonical Version'), ('6 Tre G', 'music.artist.genre', 'Hip hop music'), ('Jaxon Bieber', 'common.topic.notable_types', 'Person'), ('Beyoncé Knowles', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('m.0gbm3d_', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('m.0_xlw5f', 'award.award_nomination.nominated_for', 'Wait for a Minute'), ('m.0gc_9w6', 'common.webpage.category', 'Video Stream Host Page'), ('m.0nfnx_3', 'film.personal_film_appearance.person', 'Justin Bieber'), ('We Are the World 25 for Haiti', 'music.album.contributor', 'm.0vpggkk'), ('Dapo Torimiro', 'people.person.profession', 'Singer'), ('Fabolous', 'people.person.nationality', 'United States of America'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'EPMD'), ('Christina Aguilera', 'people.person.profession', 'Record producer'), ('Taylor Swift', 'people.person.nationality', 'United States of America'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1w0t'), ('First Dance', 'common.topic.article', 'm.0g5q9l1'), ('Chris Brown', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Mary J. Blige', 'music.artist.genre', 'Rhythm and blues'), ('Rihanna', 'broadcast.artist.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('FLOW 103', 'broadcast.content.artist', 'Akon'), ('2012 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0njw4z2'), ('Confident', 'common.topic.notable_for', 'g.1yp3bsmrd'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z1ndbl'), ('m.0101ft2v', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Flo Rida'), ('Young Artists for Haiti', 'common.topic.notable_types', 'Musical Artist'), ('Jennifer Lopez', 'people.person.profession', 'Actor'), ('Jordin Sparks', 'music.artist.genre', 'Pop music'), ('Ricky Nelson', 'music.artist.genre', 'Pop music'), ('2013 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0wjgrzc'), ('HitzRadio.com', 'common.topic.image', 'HitzRadio.com'), ('Boyfriend', 'music.composition.composer', 'Mason Levy'), ('Boyfriend (Dada Life remix)', 'music.recording.song', 'Boyfriend'), ('Tampa', 'location.hud_county_place.place', 'Tampa'), ('Jessie J', 'music.artist.genre', 'Contemporary R&B'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Initial release date'), ('Colbie Caillat', 'music.artist.genre', 'Acoustic music'), ('m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Year'), ('Snoop Dogg', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber', 'music.featured_artist.recordings', 'Beautiful'), ('Justin Bieber', 'music.artist.origin', 'Stratford'), ('Rhythm and blues', 'broadcast.genre.content', 'radioIO RNB Mix'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never'), ('m.0z340zt', 'freebase.valuenotation.is_reviewed', 'Year'), ('Date of birth', 'rdf-schema#domain', 'Person'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'TLC'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Chris Brown'), ('Pop music', 'base.musicpf.genre.artist', 'Yves Bole'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me (album version)'), ('FLOW 103', 'broadcast.content.artist', 'Baby Bash'), ('m.0_srv2b', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Jordan Francis', 'people.person.profession', 'Actor'), ('Christina Aguilera', 'music.artist.genre', 'Rock music'), ('Justin Timberlake', 'people.person.profession', 'Singer'), ('1.FM Top 40', 'broadcast.content.artist', 'Taylor Swift'), ('Justin Bieber', 'people.person.profession', 'Dancer'), ('m.0z2dr9y', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'International'), ('My World', 'music.album.releases', 'My World'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.canonical_version', 'Boyfriend'), ('Dance music', 'broadcast.genre.content', '1Club.FM: 80s (Pop)'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Spouse (or domestic partner)'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.artist', 'Justin Bieber'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Nickelback'), ('Big R Radio - Top 40 Hits', 'broadcast.content.producer', 'Big R Radio Network'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Fabolous', 'broadcast.artist.content', '.977 The Hits Channel'), ('Brandy Norwood', 'music.artist.genre', 'Pop music'), ('Chicago', 'location.administrative_division.country', 'United States of America'), ('Khalil', 'celebrities.celebrity.celebrity_friends', 'm.012r2v_0'), ('Bryan Adams', 'people.person.profession', 'Record producer'), ('Cory Gunz', 'music.artist.genre', 'Hip hop music'), ('Nasri', 'music.composer.compositions', 'All Around The World'), ('Person or entity appearing in film', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beauty and a Beat', 'music.single.versions', 'Beauty and a Beat (Bisbetic Remix)'), ('m.0y7y53r', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.0z8t2dy', 'award.award_nomination.award', 'NME Award for Worst Album'), ('Katy Perry', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Lolly', 'music.recording.song', 'Lolly'), ('Timbaland', 'broadcast.artist.content', '1Club.FM: Power'), ('Rob Thomas', 'music.artist.genre', 'Rock music'), ('Everlast', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chris Brown'), ('Barry Weiss', 'people.person.gender', 'Male'), ('CL', 'music.artist.genre', 'Pop music'), ('FLOW 103', 'broadcast.content.artist', 'The Black Eyed Peas'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mat Silver & Tony Burt'), ('My World', 'music.album.releases', 'My World'), ('Ashlee Simpson', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long As You Love Me (Ferry Corsten radio)', 'music.recording.song', 'As Long as You Love Me'), ('Hot Wired Radio', 'common.topic.notable_types', 'Broadcast Content'), ('Ice Cube', 'people.person.profession', 'Actor'), ('Boyfriend', 'music.single.versions', 'Boyfriend (Neon NiteClub Remix)'), ('m.0_xlw5f', 'award.award_nomination.award_nominee', 'Tyga'), ('Boyfriend', 'common.topic.article', 'm.0j64n8h'), ('Beauty and a Beat', 'music.album.genre', 'Electronic dance music'), ('Leif Garrett', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmxj'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0tkqqgg'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('David Nicksay', 'film.producer.films_executive_produced', 'Justin Bieber: Never Say Never'), ('Musical Artist', 'type.type.expected_by', 'Artist'), ('Pras', 'music.artist.genre', 'Contemporary R&B'), ('Justin Bieber', 'music.artist.album', 'Heartbreaker'), ('Janet Jackson', 'music.artist.genre', 'Hip hop music'), ('m.0101ft56', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('radioIO Todays RNB', 'broadcast.content.genre', 'Rhythm and blues'), ('Ciara', 'broadcast.artist.content', '.977 The Hits Channel'), ('Demi Lovato', 'music.artist.genre', 'Rock music'), ('DMX', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('All That Matters', 'common.topic.notable_for', 'g.1yl5vjhm6'), ('Colbie Caillat', 'people.person.nationality', 'United States of America'), ('m.0n4rmg7', 'award.award_honor.honored_for', 'U Smile'), ('FLOW 103', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Height'), ('My Worlds Acoustic', 'music.album.release_type', 'Album'), ('m.0v90skf', 'freebase.valuenotation.has_no_value', 'Winning work'), ('Lolly', 'music.recording.canonical_version', 'Lolly'), ('Justin Bieber: First Step 2 Forever', 'common.topic.notable_for', 'g.1yl5jc8_8'), ('m.0yrk18w', 'award.award_honor.award_winner', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Crazy Town'), ('Lady Antebellum', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Love Never Felt So Good', 'music.album.artist', 'Justin Bieber'), ('Avril Lavigne', 'music.artist.genre', 'Teen pop'), ('Ray J', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Colbie Caillat', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miami Beach', 'location.hud_county_place.place', 'Miami Beach'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Somebody to Love (remix)', 'music.album.artist', 'Justin Bieber'), ('Adrienne Bailon', 'music.artist.genre', 'Contemporary R&B'), ('m.0njhvsq', 'freebase.valuenotation.is_reviewed', 'Year'), ('m.0vp7cl4', 'music.recording_contribution.album', 'Somebody to Love (remix)'), ('Shaun Cassidy', 'people.person.profession', 'Film Producer'), ('Cris Cab', 'music.artist.label', 'The Island Def Jam Music Group'), ('m.0yqsnwz', 'organization.organization_relationship.parent', 'The Island Def Jam Music Group'), ('Christian Beadles', 'people.person.gender', 'Male'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('m.0z3tqqt', 'award.award_nomination.nominated_for', 'justinbieber'), ('WildFMRadio.com', 'broadcast.content.artist', 'Ludacris'), ('Jay-Z', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Trick Daddy', 'music.artist.origin', 'Miami'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Baby Bash'), ('Jordan Francis', 'music.artist.genre', 'Pop music'), ('m.0pc67ly', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0njgyk4', 'award.award_honor.honored_for', 'My World 2.0'), ('Whitney Houston', 'influence.influence_node.influenced', 'Lady Gaga'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Raekwon', 'music.artist.contribution', 'm.0vp7m_x'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Kanye West'), ('Kuk Harrell', 'common.topic.notable_types', 'Record Producer'), ('Urban contemporary', 'broadcast.genre.content', 'HitzRadio.com'), ('Justin Bieber', 'music.composer.compositions', 'Somebody to Love'), ('Blu Cantrell', 'people.person.nationality', 'United States of America'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Coldplay', 'common.topic.notable_types', 'Musical Artist'), ('Kid Cudi', 'people.person.profession', 'Actor'), ('Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Official website'), ('HitzRadio.com', 'broadcast.content.artist', 'Flobots'), ('Soulja Boy', 'people.person.profession', 'Actor'), ('Record producer', 'base.sounds.audio_recording_role.recordings', 'm.09jx30f'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mariah Carey'), ('The Pussycat Dolls', 'music.artist.genre', 'Pop music'), ('Pattie Mallette', 'common.topic.notable_types', 'Author'), ('Recovery', 'music.composition.composer', 'Dominic Jordan'), ('New Kids on the Block', 'music.artist.genre', 'Hip hop music'), ('Taylor Swift', 'music.artist.genre', 'Teen pop'), ('HitzRadio.com', 'broadcast.content.artist', 'September'), ('Singer', 'common.topic.notable_types', 'Profession'), ('Alanis Morissette', 'people.person.profession', 'Writer'), ('Aaliyah', 'music.artist.genre', 'Dance-pop'), ('R. Kelly', 'music.artist.origin', 'Chicago'), ('Baby', 'music.composition.recorded_as_album', 'Baby'), ('Lolly', 'music.composition.recordings', 'Lolly'), ('m.012bm2v1', 'celebrities.friendship.friend', 'Yves Bole'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'people.profession.specialization_of', 'Musician'), ('RBMG Records', 'common.topic.notable_types', 'Record label'), ('Michael Jackson', 'broadcast.artist.content', 'GotRadio - RnB Classics'), ('Katy Perry', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Justin Bieber Fan Club', 'common.resource.annotations', 'm.0dj34q5'), ('Annette Funicello', 'people.person.nationality', 'United States of America'), ('Usher', 'common.topic.notable_types', 'Musical Artist'), ('#thatPOWER', 'common.topic.notable_for', 'g.1yg9cvthd'), ('Jeremy Bieber', 'people.person.children', 'Jaxon Bieber'), ('m.0101fv7x', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('m.0z340zt', 'award.award_honor.ceremony', '2012 MTV Europe Music Awards'), ('Guest host', 'tv.non_character_role.tv_guest_personal_appearances', 'm.0v_72js'), ('Justin Bieber', 'music.lyricist.lyrics_written', 'Bigger'), ('m.01053qzf', 'film.personal_film_appearance.person', 'Jeremy Bieber'), ('2013 MTV Europe Music Awards', 'award.award_ceremony.awards_presented', 'm.0yrhmll'), ('Heartbreaker', 'award.award_nominated_work.award_nominations', 'm.0y80n25'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Rihanna'), ('Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Taylor Swift'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Ludacris'), ('MTV Europe Music Award for Best Canadian Act', 'award.award_category.winners', 'm.0yqfny6'), ('Timbaland', 'people.person.nationality', 'United States of America'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend (acoustic version)'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Pras', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Judy Garland', 'people.person.gender', 'Female'), ('Toby Gad', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Diplo', 'music.artist.genre', 'Hip hop music'), ('Ice Cube', 'people.person.profession', 'Record producer'), ('Bryan-Michael Cox', 'people.person.gender', 'Male'), ('My World', 'music.album.genre', 'Dance-pop'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.01074ns9'), ('School Boy Records', 'music.record_label.artist', 'CL'), ('Eenie Meenie', 'common.topic.notable_for', 'g.1yp3c7ts1'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrk4gn'), ('Taylor Swift', 'freebase.valuenotation.has_no_value', 'Children'), ('HitzRadio.com', 'broadcast.content.artist', 'Baby Bash'), ('Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Demi Lovato', 'people.person.gender', 'Female'), ('Pras', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Usher', 'film.producer.film', \"Justin Bieber's Believe\"), ('Justin Bieber', 'people.person.profession', 'Singer'), ('My Worlds Acoustic', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Area codes 519 and 226', 'location.location.contains', 'London'), ('Lil Wayne', 'people.person.gender', 'Male'), (\"2013 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgkrj4'), ('Chicago', 'location.location.containedby', 'United States of America'), ('Usher', 'broadcast.artist.content', '.977 The Hits Channel'), ('#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'), ('1.FM Top 40', 'broadcast.content.artist', 'Alanis Morissette'), ('m.0_cz5l3', 'celebrities.legal_entanglement.celebrity', 'Justin Bieber'), ('Film', 'freebase.type_profile.strict_included_types', 'Topic'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0j2gfc1'), ('m.0gj1yzm', 'tv.tv_guest_role.episodes_appeared_in', 'Series 1, Show 4'), ('1Club.FM: Power', 'broadcast.content.artist', 'Kanye West'), ('Next to You', 'music.recording.featured_artists', 'Justin Bieber'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Enrique Iglesias', 'people.person.nationality', 'United States of America'), ('Boyfriend (Neon NiteClub Remix)', 'music.recording.song', 'Boyfriend'), ('Anastacia', 'music.artist.origin', 'Chicago'), ('Fergie', 'people.person.profession', 'Musician'), ('Usher', 'music.artist.contribution', 'm.0vp7cl4'), ('Justin Bieber', 'people.person.places_lived', 'm.0kt8f87'), ('Ronald Isley', 'music.artist.genre', 'Rhythm and blues'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Dance music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0z87d3n'), ('Live My Life', 'music.composition.recordings', 'Live My Life'), ('Nicki Minaj', 'people.person.profession', 'Musician'), ('Chris Jasper', 'people.person.profession', 'Singer-songwriter'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft56'), ('Favorite Girl', 'common.topic.notable_for', 'g.12h323s38'), ('Die in Your Arms', 'common.topic.notable_for', 'g.1yh9szjwm'), ('Hip hop music', 'broadcast.genre.content', '#Musik.Main on RauteMusik.FM'), ('Singer', 'common.topic.notable_for', 'g.12557cdvn'), ('Lady Gaga', 'people.person.profession', 'Record producer'), ('Diplo', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('All Bad', 'music.recording.artist', 'Justin Bieber'), ('2011 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0njhxzc'), ('m.0gbcs16', 'tv.tv_guest_role.episodes_appeared_in', 'August 20, 2010'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0tk76mf'), ('Contemporary R&B', 'broadcast.genre.content', 'WildFMRadio.com'), ('Verse Simmonds', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Eenie Meenie', 'music.album.artist', 'Sean Kingston'), ('Jay-Z', 'broadcast.artist.content', '1Club.FM: Power'), ('m.0_cyzs_', 'celebrities.legal_entanglement.celebrity', 'Justin Bieber'), ('The Pussycat Dolls', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Teen idol', 'common.topic.webpage', 'm.09wnjlc'), ('Annette Funicello', 'people.person.profession', 'Actor'), ('Live My Life', 'music.recording.canonical_version', 'Live My Life'), ('Beauty and a Beat (Bisbetic Remix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Country', 'freebase.type_hints.included_types', 'Topic'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0ng_k21'), ('Lupe Fiasco', 'common.topic.notable_types', 'Musical Artist'), ('Drake', 'music.featured_artist.albums', 'Right Here'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'AnnaGrace'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Sheryl Crow', 'people.person.profession', 'Actor'), ('Pop music', 'music.genre.subgenre', 'Synthpop'), ('FLOW 103', 'broadcast.content.genre', 'Contemporary R&B'), ('Justin Bieber: Never Say Never', 'film.film.edited_by', 'Jay Cassidy'), ('Dan Cutforth', 'film.director.film', 'Katy Perry: Part of Me'), ('Somebody to Love', 'music.recording.artist', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjpm'), ('m.0_xlw5f', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0_sxby0', 'award.award_nomination.nominated_for', 'justinbieber'), ('m.0wjgqck', 'award.award_honor.award', 'Teen Choice Award for Choice Single: Male Artist'), ('Selena Gomez', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('#Thatpower', 'music.recording.tracks', '#thatPOWER'), ('Lady Gaga', 'people.person.profession', 'Actor'), ('GotRadio - RnB Classics', 'broadcast.content.language', 'English Language'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Sir Mix-a-Lot', 'people.person.profession', 'Musician'), ('Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Will Smith', 'people.person.nationality', 'United States of America'), ('The Notorious B.I.G.', 'broadcast.artist.content', '.977 The Hits Channel'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Contemporary R&B'), ('m.0101fvhv', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Christian Beadles', 'celebrities.celebrity.celebrity_friends', 'm.0d33hsj'), ('m.0gbm3fr', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Chicago', 'common.topic.notable_types', 'City/Town/Village'), ('HitzRadio.com', 'broadcast.content.artist', 'Natasha Bedingfield'), ('Justin Bieber', 'music.artist.album', 'One Less Lonely Girl'), ('My Worlds', 'music.album.release_type', 'Album'), ('CL', 'music.artist.genre', 'Rhythm and blues'), ('CL', 'people.person.profession', 'Singer'), ('Usher', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Twista', 'broadcast.artist.content', '181-beat'), ('Dr. Dre', 'people.person.gender', 'Male'), ('Christina Aguilera', 'music.artist.genre', 'Synthpop'), ('Contemporary R&B', 'broadcast.genre.content', 'radioIO Todays RNB'), ('Duffy', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Brandy Norwood', 'people.person.profession', 'Film Producer'), ('Live My Life', 'music.composition.composer', 'Justin Bieber'), ('HitzRadio.com', 'broadcast.content.genre', 'Hip hop music'), ('m.0v_70fx', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Place of birth', 'type.property.expected_type', 'Location'), ('Kuk Harrell', 'people.person.nationality', 'United States of America'), ('Beauty and a Beat', 'music.album.genre', 'Synthpop'), ('Juno Award for Pop Album of the Year', 'award.award_category.nominees', 'm.0sxhq3d'), ('Shaun Cassidy', 'base.icons.icon.icon_genre', 'Teen idol'), ('Usher', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Montell Jordan', 'people.person.languages', 'English Language'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Panic! at the Disco'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('m.0n58kgb', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0v_72tb', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Vanessa Hudgens', 'music.artist.genre', 'Pop music'), ('As Long as You Love Me', 'music.composition.recordings', 'As Long As You Love Me (Audiobot instrumental)'), ('Verse Simmonds', 'people.person.nationality', 'United States of America'), ('Hot 108 Jamz', 'broadcast.content.genre', 'Pop music'), ('#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'), ('m.0njw257', 'award.award_honor.award_winner', 'Justin Bieber'), ('Believe', 'common.topic.image', 'justin bieber'), ('m.0zbg2kw', 'award.award_nomination.award', 'American Music Award for Favorite Pop/Rock Album'), ('m.0_vmmj6', 'award.award_nomination.nominated_for', 'Beauty And A Beat'), ('m.0n4rmg7', 'award.award_honor.ceremony', '2011 MTV Video Music Awards'), ('Ciara', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Somebody to Love', 'music.composition.recordings', 'Somebody to Love (J Stax remix)'), ('m.0gbm3b7', 'film.personal_film_appearance.film', 'Justin Bieber: Never Say Never'), ('Beautiful', 'common.topic.notable_types', 'Musical Recording'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Weezer'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Iggy Azalea'), ('1Club.FM: Mix 106', 'common.topic.image', '1clubfm.jpg'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ginuwine', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftp1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0ywvh8k'), ('Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Michael Jackson', 'broadcast.artist.content', \"1Club.FM: Jammin' Oldies\"), ('Anastacia', 'music.artist.genre', 'Dance music'), ('Confident', 'music.composition.composer', 'Justin Bieber'), ('Nasri', 'people.person.nationality', 'Canada'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0j8z6tl'), ('m.0101fs_z', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Donna Summer', 'broadcast.artist.content', 'radioIO Classic RNB'), ('Usher', 'people.person.profession', 'Singer-songwriter'), ('Teen idol', 'base.icons.icon_genre.icons', 'Justin Bieber'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Zac Hanson'), ('Ja Rule', 'broadcast.artist.content', '.977 The Hits Channel'), ('Sir Nolan', 'music.composer.compositions', 'All Around The World'), ('All Around the World', 'music.single.versions', 'All Around the World'), ('Singer', 'people.profession.corresponding_type', 'Musical Artist'), ('Sia Furler', 'common.topic.notable_types', 'Musical Artist'), ('The Notorious B.I.G.', 'people.person.profession', 'Musician'), ('Hikaru Utada', 'music.artist.genre', 'Dance music'), ('Jordan Francis', 'music.artist.genre', 'Synthpop'), ('P!nk', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fergie'), ('m.0gj0bj6', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Donna Summer', 'music.artist.genre', 'Disco'), ('Savan Kotecha', 'music.artist.genre', 'Contemporary R&B'), ('Pray', 'common.topic.notable_for', 'g.1yp3d4v1g'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Parents'), ('Bad Day', 'music.recording.song', 'Bad Day'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0130c918'), (\"PartyRadioUSA.net - Nation's #1 Party Authority\", 'broadcast.content.genre', 'Trance music'), ('Rock music', 'common.topic.subject_of', 'Steve Blevins'), ('#thatPower', 'music.composition.composer', 'Damien LeRoy'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'ABBA'), ('Artist', 'people.profession.specializations', 'Musician'), ('Record producer', 'common.topic.webpage', 'm.09xh0kp'), ('Hot Wired Radio', 'broadcast.content.artist', 'Pearl Jam'), ('Justin Bieber', 'music.featured_artist.albums', 'Live My Life'), ('One Less Lonely Girl', 'common.topic.notable_for', 'g.126sh6292'), ('As Long as You Love Me', 'music.album.album_content_type', 'Remix album'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bigger', 'common.topic.notable_types', 'Composition'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5tj13'), ('#thatPower', 'common.topic.notable_types', 'Composition'), ('.977 The Hits Channel', 'broadcast.content.artist', 'DMX'), ('m.0vp7m_x', 'music.recording_contribution.album', 'Runaway Love (remix)'), ('Die in Your Arms', 'music.composition.composer', 'Justin Bieber'), ('JoJo', 'broadcast.artist.content', '.977 The Hits Channel'), ('PowerHitz', 'broadcast.content.artist', 'Nelly'), ('m.0z85n_1', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Baby', 'common.topic.notable_types', 'Musical Recording'), ('Hot Wired Radio', 'broadcast.content.artist', 'Matchbox Twenty'), ('m.0yrlqp1', 'education.education.institution', 'Stratford Northwestern Secondary School'), ('Lolly', 'music.recording.artist', 'Maejor'), ('Somebody to Love', 'music.recording.song', 'Somebody to Love'), ('My Worlds: The Collection', 'music.album.genre', 'Contemporary R&B'), ('Music executive', 'common.topic.notable_types', 'Profession'), ('Person', 'type.type.properties', 'Profession'), ('FLOW 103', 'broadcast.content.artist', 'Usher'), ('All Around the World', 'music.album.contributor', 'm.0vp800w'), ('Spanish Language', 'language.human_language.countries_spoken_in', 'Canada'), ('Ginuwine', 'people.person.profession', 'Singer-songwriter'), ('Sean Combs', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Sir Mix-a-Lot', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Justin Bieber: Never Say Never', 'film.film.estimated_budget', 'm.0h17ns7'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'The Pussycat Dolls'), ('1Club.FM: Mix 106', 'broadcast.content.genre', 'Classic hits'), ('We Are the World 25 for Haiti', 'music.album.album_content_type', 'Studio album'), ('Nelly', 'people.person.profession', 'Actor'), ('50 Cent', 'people.person.gender', 'Male'), ('Wait for a Minute', 'music.composition.composer', 'Jess Jackson'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvm4'), ('Sean Combs', 'people.person.gender', 'Male'), ('Juvenile', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Beautiful', 'common.topic.article', 'm.0pc5x25'), ('Michael Jackson', 'influence.influence_node.influenced', 'Usher'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Carl B'), ('Terius Nash', 'broadcast.artist.content', 'PowerHitz'), ('Selena Gomez', 'people.person.gender', 'Female'), ('Nathan Lanier', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Record producer', 'base.ontologies.ontology_instance.equivalent_instances', 'm.09klxxt'), ('Shaffer Smith', 'broadcast.artist.content', 'Sunshine Radio'), ('JellyRadio.com', 'broadcast.content.artist', 'Dr. Dre'), ('m.0n4rmg7', 'award.award_honor.award', 'MTV Video Music Award for Best Male Video'), ('Canadian', 'base.firsts.first_achievement_category.firsts', 'm.0j1t2xs'), ('Baby', 'award.award_winning_work.awards_won', 'm.0njhyh_'), ('1Club.FM: Power', 'common.topic.image', '1clubfm.jpg'), ('Juvenile', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Tyga', 'people.person.gender', 'Male'), ('Sir Nolan', 'people.person.profession', 'Record producer'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Mario'), ('Lil Wayne', 'music.artist.genre', 'Rock music'), ('Lupe Fiasco', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'common.topic.subjects', 'Singer'), ('My Worlds: The Collection', 'music.album.genre', 'Dance-pop'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Gender'), ('All Bad', 'music.composition.recordings', 'All Bad'), ('Jessica Simpson', 'people.person.nationality', 'United States of America'), ('Singer', 'people.profession.specializations', 'Jazz Artist/Singer'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Wait For a Minute', 'music.recording.featured_artists', 'Justin Bieber'), ('Hold Tight', 'common.topic.notable_types', 'Composition'), ('LeAnn Rimes', 'music.artist.genre', 'Pop music'), ('m.0v1d2xz', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('m.0k05qcr', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Chingy', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0gxnp02', 'base.popstra.dated.participant', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', '50 Cent'), ('HitzRadio.com', 'broadcast.content.artist', 'Pras'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Sean Kingston'), ('1Club.FM: Power', 'broadcast.content.location', 'Chicago'), ('R. Kelly', 'people.person.profession', 'Actor'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Emphatic Radio.com!', 'broadcast.content.genre', 'Pop music'), ('m.0101gx29', 'people.marriage.spouse', 'Jeremy Bieber'), ('HitzRadio.com', 'broadcast.content.artist', 'The All-American Rejects'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Contemporary R&B'), ('Winning work', 'rdf-schema#range', 'Award-Winning Work'), ('Jayceon Terrell Taylor', 'people.person.profession', 'Musician'), ('U Smile', 'music.album.release_type', 'Single'), ('Kevin Risto', 'music.composer.compositions', 'Bigger'), ('Juvenile', 'broadcast.artist.content', 'PowerHitz'), ('Dapo Torimiro', 'people.person.profession', 'Record producer'), ('Enrique Iglesias', 'music.artist.genre', 'Contemporary R&B'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvfs'), ('Runaway Love (remix)', 'music.recording.featured_artists', 'Justin Bieber'), ('Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me', 'music.album.releases', 'As Long as You Love Me'), ('Britney Spears', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Reggae', 'music.genre.parent_genre', 'World music'), ('Benny Blanco', 'people.person.gender', 'Male'), ('All That Matters', 'music.composition.composer', 'Dre & Vidal'), ('Brandy Norwood', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Rihanna', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Ice Cube', 'freebase.valuenotation.is_reviewed', 'Official website'), ('Next to You', 'music.album.featured_artists', 'Justin Bieber'), ('m.0101fv74', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Jessica Jarrell'), ('RBMG Records', 'common.topic.article', 'm.0gmfc7y'), ('P!nk', 'people.person.profession', 'Model'), ('Usher', 'music.artist.genre', 'Dance-pop'), ('Jeremy Bieber', 'people.person.place_of_birth', 'Stratford'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_1wm'), ('Lil Jon', 'people.person.languages', 'English Language'), ('Indie rock', 'music.genre.parent_genre', 'Rock music'), ('1.FM Top 40', 'broadcast.content.artist', 'Goldfrapp'), ('m.0v_72js', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Justin Bieber: Never Say Never', 'film.film.genre', 'Rockumentary'), ('All Around The World', 'music.composition.recordings', 'All Around the World'), ('Never Say Never', 'music.album.artist', 'Justin Bieber'), ('m.0ncpx3v', 'organization.leadership.person', 'Barry Weiss'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Britney Spears'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Profession'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvp7'), ('Sean Combs', 'people.person.profession', 'Actor'), ('Barry Weiss', 'people.person.nationality', 'United States of America'), ('Synthpop', 'music.genre.parent_genre', 'Synthpop'), ('Rodney Jerkins', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Justin Bieber', 'people.person.sibling_s', 'm.0gxnnwp'), ('Thought Of You', 'music.composition.recordings', 'Thought of You'), ('Akon', 'broadcast.artist.content', 'FLOW 103'), ('Toby Gad', 'people.person.profession', 'Musician'), ('Rob Thomas', 'people.person.profession', 'Singer'), ('Teen Choice Award for Choice Music: Album Pop', 'award.award_category.winners', 'm.0z8qx3w'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.010lkp2z', 'award.award_honor.award_winner', 'Justin Bieber'), ('Gavin DeGraw', 'people.person.profession', 'Musician'), ('Reed Smoot', 'freebase.valuenotation.has_value', 'Parents'), ('Katy Perry: Part of Me', 'film.film.directed_by', 'Dan Cutforth'), ('Yves Bole', 'freebase.valuenotation.has_no_value', 'Notable types'), ('Enrique Iglesias', 'people.person.profession', 'Model'), ('Donna Summer', 'broadcast.artist.content', 'BeirutNights.com Radio'), ('Shaun Cassidy', 'people.person.nationality', 'United States of America'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0rqh7d9'), ('Sean Combs', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('PowerHitz', 'broadcast.content.artist', 'Rihanna'), ('m.0njhtjj', 'award.award_honor.award_winner', 'Justin Bieber'), ('Terence Dudley', 'music.artist.genre', 'Rhythm and blues'), ('Lady Gaga', 'people.person.profession', 'Film Producer'), ('Robby Stewart', 'fictional_universe.fictional_character.occupation', 'Record producer'), ('m.0z3vx9q', 'music.music_video_performance.special_music_video_performance_type', 'Dancer'), ('All That Matters', 'music.album.artist', 'Justin Bieber'), ('Snoop Dogg', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Dapo Torimiro', 'people.person.gender', 'Male'), ('Musical Album', 'freebase.type_profile.strict_included_types', 'Topic'), ('Beautiful', 'music.recording.song', 'Beautiful'), ('Never Let You Go', 'music.composition.lyricist', 'Bryan-Michael Cox'), ('m.043082k', 'common.webpage.topic', '1Club.FM: Channel One'), ('PowerHitz', 'broadcast.content.genre', 'Pop music'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('JellyRadio.com', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Beyoncé Knowles', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntw'), ('P!nk', 'people.person.profession', 'Singer-songwriter'), ('Don Henley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Urban contemporary', 'broadcast.genre.content', 'WildFMRadio.com'), ('m.0jvgmxc', 'award.award_honor.award', 'Billboard Music Award for Top Social Artist'), ('Singer', 'music.music_video_character.portrayed_in_music_videos', 'm.0j21b47'), ('Ciara', 'broadcast.artist.content', 'WildFMRadio.com'), ('Amerie', 'people.person.profession', 'Record producer'), ('Urban contemporary', 'broadcast.genre.content', 'Emphatic Radio.com!'), ('m.010lkp2z', 'freebase.valuenotation.is_reviewed', 'Award category'), ('1.FM Top 40', 'broadcast.content.artist', 'Maroon 5'), ('Will i Am', 'people.person.profession', 'Singer'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Parents'), ('m.0101fsz2', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Justin Bieber: Never Say Never', 'film.film.costume_design_by', 'Kurt Swanson'), ('Alicia Keys', 'broadcast.artist.content', '1Club.FM: V101'), ('Beyoncé Knowles', 'broadcast.artist.content', 'FLOW 103'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me'), ('Beauty And A Beat', 'music.composition.language', 'English Language'), ('Change Me', 'music.album.release_type', 'Single'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Usher'), ('Sean Kingston', 'music.artist.origin', 'Miami'), ('Teen idol', 'common.topic.webpage', 'm.09xx941'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flo Rida'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Under the Mistletoe', 'award.award_winning_work.awards_won', 'm.0z8755b'), ('Yves Bole', 'freebase.valuenotation.has_no_value', 'Image'), ('Selena Gomez', 'base.popstra.celebrity.dated', 'm.0gxnp02'), ('Boyfriend', 'music.single.versions', 'Boyfriend (acoustic version)'), ('Jordan Pruitt', 'people.person.nationality', 'United States of America'), ('Lady Gaga', 'music.artist.genre', 'Dance-pop'), ('Runaway Love (remix)', 'common.topic.notable_types', 'Musical Album'), ('Beauty and a Beat (Remixes)', 'music.album.primary_release', 'Beauty and a Beat (Remixes)'), ('Hot Wired Radio', 'broadcast.content.artist', 'Red Hot Chili Peppers'), ('Rihanna', 'people.person.profession', 'Actor'), ('Mary J. Blige', 'people.person.profession', 'Singer-songwriter'), ('Kylie Minogue', 'broadcast.artist.content', '1.FM Top 40'), ('Ice Cube', 'people.person.nationality', 'United States of America'), ('Musical Album', 'type.type.properties', 'Artist'), ('Blu Cantrell', 'music.artist.genre', 'Rhythm and blues'), ('Beauty and a Beat (Wideboys Dub)', 'common.topic.notable_types', 'Musical Recording'), ('m.0y7y53r', 'freebase.valuenotation.is_reviewed', 'Winning work'), ('Montell Jordan', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'K-Ci & JoJo'), (\"Destiny's Child\", 'broadcast.artist.content', 'radioIO RNB Mix'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chingy'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The All-American Rejects'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('iJustine', 'influence.influence_node.influenced_by', 'Yves Bole'), ('m.0_cyzs_', 'celebrities.legal_entanglement.location', 'Miami'), ('Judy Garland', 'people.person.languages', 'English Language'), (\"O'Kelly Isley, Jr.\", 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Sia Furler', 'people.person.profession', 'Singer'), ('m.0101ft35', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('m.0v_729v', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Shaffer Smith', 'people.person.gender', 'Male'), ('radioIO Classic RNB', 'broadcast.content.genre', 'Rhythm and blues'), ('Terence Dudley', 'people.person.nationality', 'United States of America'), ('m.0gbm3g2', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Singer', 'people.profession.specializations', 'Opera Singer'), ('m.012bq275', 'celebrities.friendship.friend', 'Justin Bieber'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gttv'), ('Boyfriend', 'music.album.releases', 'Boyfriend'), ('Deborah Lurie', 'film.music_contributor.film', 'Justin Bieber: Never Say Never'), ('Ja Rule', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0zg2w8r', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('m.0f0dwc4', 'common.webpage.category', 'Curated Topic'), ('.977 The Hits Channel', 'broadcast.content.artist', 'The Notorious B.I.G.'), ('Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5m'), ('JoJo', 'common.topic.notable_types', 'Musical Artist'), ('m.046c33q', 'common.webpage.topic', 'Emphatic Radio.com!'), ('MTV Movie Award for Best Jaw Dropping Moment', 'award.award_category.nominees', 'm.0pc67ly'), ('Eenie Meenie', 'music.composition.composer', 'Benny Blanco'), ('m.0y_g42w', 'award.award_nomination.award', 'Shorty Award for Celebrity'), ('Journals', 'common.topic.notable_types', 'Musical Album'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPower'), ('Believe', 'music.album.genre', 'Hip hop music'), ('Musician', 'people.profession.specialization_of', 'Artist'), (\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2v'), ('Jaden Smith', 'people.person.nationality', 'United States of America'), ('Bigger', 'music.composition.lyricist', 'Waynne Nugent'), ('#thatPOWER', 'music.recording.artist', 'Justin Bieber'), ('Justin Bieber', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Beauty and a Beat (Wideboys Club Mix)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('JellyRadio.com', 'broadcast.content.genre', 'Top 40'), ('m.0z3vx9q', 'music.music_video_performance.special_music_video_performance_type', 'Actor'), ('All Around the World', 'music.single.versions', 'All Around The World (featuring Ludacris)'), ('Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Estelle', 'music.artist.genre', 'Hip hop music'), ('JoJo', 'music.artist.genre', 'Contemporary R&B'), ('Leonardo DiCaprio', 'people.person.profession', 'Actor'), ('Somebody To Love', 'music.single.versions', 'Somebody to Love (J Stax remix)'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4r4'), ('Kuk Harrell', 'music.artist.genre', 'Pop music'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0y_g42w'), ('m.012r2v_0', 'celebrities.friendship.friend', 'Khalil'), ('Yves Bole', 'people.person.gender', 'Male'), ('Runaway Love (remix)', 'music.album.featured_artists', 'Raekwon'), ('m.0ndc259', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z3tqqt'), ('m.0sxhq3d', 'award.award_nomination.award', 'Juno Award for Pop Album of the Year'), ('PowerHitz', 'broadcast.content.artist', 'Baby Bash'), ('Elvis Presley', 'music.artist.genre', 'Rhythm and blues'), ('Clay Aiken', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.0n58kgb', 'award.award_nomination.nominated_for', 'Believe'), ('m.0pbzq13', 'film.performance.actor', 'Justin Bieber'), ('Yves Bole', 'music.artist.genre', 'Pop music'), ('Sunshine Radio', 'common.topic.article', 'm.03hr_cl'), ('Raekwon', 'people.person.nationality', 'United States of America'), ('Chloe Bridges', 'people.person.gender', 'Female'), ('Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0z1jn32'), ('Will i Am', 'people.person.profession', 'Actor'), ('Rita Ora', 'freebase.valuenotation.has_no_value', 'Children'), ('m.09klxxt', 'base.ontologies.ontology_instance_mapping.freebase_topic', 'Record producer'), ('Fabolous', 'common.topic.notable_types', 'Musical Artist'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmnvf'), ('Britney Spears', 'people.person.profession', 'Actor'), ('As Long As You Love Me (Ferry Corsten remix)', 'common.topic.notable_types', 'Musical Recording'), ('Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Michael Jackson', 'music.artist.genre', 'Dance music'), ('Journals', 'music.album.album_content_type', 'Compilation album'), ('Somebody to Love', 'music.recording.featured_artists', 'Usher'), ('Akon', 'music.artist.genre', 'Dance music'), ('Adam Messinger', 'music.artist.genre', 'Pop music'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvj9'), ('Yves Bole', 'celebrities.celebrity.celebrity_rivals', 'm.012bm5cg'), ('Alanis Morissette', 'music.artist.genre', 'Dance-pop'), ('Boyfriend', 'music.composition.language', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.executive_produced_by', 'David Nicksay'), ('Justin Timberlake', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('RedOne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Rodney Jerkins', 'music.artist.genre', 'Hip hop music'), ('Twista', 'people.person.place_of_birth', 'Chicago'), ('1Club.FM: Channel One', 'broadcast.content.genre', 'Rock music'), ('Benny Blanco', 'people.person.profession', 'Record producer'), ('Mary J. Blige', 'broadcast.artist.content', 'Hot 108 Jamz'), ('All Around the World', 'music.recording.artist', 'Ludacris'), ('Right Here', 'music.composition.composer', 'Hit-Boy'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Tristan Prettyman'), ('Diplo', 'music.featured_artist.recordings', 'Where Are Ü Now'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0gbm3fj'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bjql'), ('L.A. Reid', 'music.artist.genre', 'Contemporary R&B'), ('John Mamann', 'freebase.valuenotation.is_reviewed', 'Gender'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0zg2qjr'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.artist', 'Justin Bieber'), ('m.0sgkrj4', 'award.award_honor.award', \"Kids' Choice Award for Favorite Male Singer\"), ('Justin Bieber: Never Say Never', 'film.film.country', 'United States of America'), ('R. Kelly', 'people.person.profession', 'Artist'), ('m.0101fvfs', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Lolly', 'common.topic.notable_for', 'g.11b75r5d3z'), ('1.FM Top 40', 'broadcast.content.artist', 'Madonna'), ('m.0k05qcr', 'measurement_unit.dated_money_value.source', 'celebritynetworth.com'), ('DMX', 'people.person.gender', 'Male'), ('Colbie Caillat', 'people.person.gender', 'Female'), ('m.0j_tkcq', 'film.personal_film_appearance.film', \"Justin Bieber: Never Say Never - Director's Fan Cut\"), ('Believe', 'music.album.releases', 'Believe'), ('Contemporary R&B', 'broadcast.genre.content', 'SoulfulClassics.com'), ('Ludacris', 'broadcast.artist.content', 'WildFMRadio.com'), ('Enrique Iglesias', 'people.person.profession', 'Record producer'), ('1Club.FM: Mix 106', 'common.topic.notable_for', 'g.1255j19v0'), ('Nick Jonas', 'music.artist.genre', 'Dance music'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Shaffer Smith'), ('My Worlds: The Collection', 'music.album.genre', 'Pop music'), ('Baby', 'music.composition.recordings', 'Baby'), ('JellyRadio.com', 'broadcast.content.artist', 'Sean Paul'), ('Beyoncé Knowles', 'music.artist.genre', 'Teen pop'), ('All Around the World', 'music.recording.canonical_version', 'All Around the World'), (\"Justin Bieber's Believe\", 'film.film.directed_by', 'Jon M. Chu'), ('Akon', 'music.artist.genre', 'Pop music'), ('JellyRadio.com', 'broadcast.content.genre', 'Contemporary R&B'), ('Hold Tight', 'music.composition.composer', 'Dominic Jordan'), ('1.FM Top 40', 'broadcast.content.artist', 'Leona Lewis'), ('Snoop Dogg', 'people.person.profession', 'Musician'), ('Ginuwine', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Duffy', 'music.artist.genre', 'Dance-pop'), ('m.0d_hbgr', 'common.webpage.topic', 'Justin Bieber'), ('Britney Spears', 'broadcast.artist.content', '.977 The Hits Channel'), ('m.0pcr2dh', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('Never Say Never', 'music.composition.recordings', 'Never Say Never'), ('Rodney Jerkins', 'music.artist.genre', 'Rock music'), ('m.0t4s_bn', 'award.award_honor.award', 'Juno Fan Choice Award'), ('Adrienne Bailon', 'people.person.nationality', 'United States of America'), ('Image', 'rdf-schema#domain', 'Topic'), ('Nicki Minaj', 'people.person.profession', 'Singer'), ('m.0z83x9l', 'freebase.valuenotation.has_no_value', 'Winning work'), ('m.012bm2cw', 'tv.regular_tv_appearance.actor', 'Yves Bole'), ('Will Smith', 'music.artist.genre', 'Pop music'), ('Shaffer Smith', 'music.artist.genre', 'Rhythm and blues'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Charlie Cohen'), ('Love Never Felt So Good', 'music.composition.composer', 'Michael Jackson'), ('Contemporary R&B', 'broadcast.genre.content', 'Classic Soul Network'), ('Jay Cassidy', 'film.editor.film', 'Justin Bieber: Never Say Never'), ('Blu Cantrell', 'people.person.profession', 'Singer-songwriter'), ('Judy Garland', 'base.icons.icon.icon_genre', 'Teen idol'), ('Mariah Carey', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('Dan Cutforth', 'film.producer.film', 'Justin Bieber: Never Say Never'), ('Enrique Iglesias', 'music.artist.genre', 'Synthpop'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Guglielmo Scilla'), ('m.0z84kwx', 'award.award_nomination.nominated_for', 'Boyfriend'), ('Nelly Furtado', 'people.person.profession', 'Record producer'), ('Aaliyah', 'broadcast.artist.content', 'HitzRadio.com'), ('Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Profession'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Jay-Z', 'people.person.profession', 'Singer'), ('DMX', 'broadcast.artist.content', 'Smoothbeats'), ('SoulfulHipHop.com Radio', 'common.topic.notable_types', 'Broadcast Content'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Sean Kingston'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Erasure'), ('Justin Bieber', 'music.artist.genre', 'Pop music'), ('Kylie Minogue', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('Nelly', 'music.artist.genre', 'Pop music'), ('Right Here', 'common.topic.notable_for', 'g.1yl5x2jhc'), (\"Turn to You (Mother's Day Dedication)\", 'common.topic.notable_types', 'Composition'), ('Emphatic Radio.com!', 'broadcast.content.artist', '50 Cent'), ('Island Records', 'organization.organization.parent', 'm.05sp405'), ('Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9h8f'), ('Justin Bieber', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0v_714c'), ('Rita Ora', 'people.person.gender', 'Female'), ('Yves Bole', 'music.artist.album', 'You'), ('Drake', 'people.person.profession', 'Actor'), ('Blackstreet', 'music.artist.genre', 'Electronic dance music'), ('1Club.FM: 80s (Pop)', 'broadcast.content.genre', 'Contemporary R&B'), ('As Long As You Love Me (Audien dubstep mix)', 'music.recording.song', 'As Long as You Love Me'), ('181-beat', 'broadcast.content.artist', 'Beyoncé Knowles'), ('Lolly', 'music.recording.featured_artists', 'Justin Bieber'), ('Rudolph Isley', 'people.person.profession', 'Singer'), ('m.0_syttc', 'award.award_nomination.nominated_for', 'justinbieber'), ('Live My Life', 'music.composition.composer', 'Kev Nish'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Blank & Jones'), ('Never Say Never', 'music.composition.recorded_as_album', 'Never Say Never'), ('Anastacia', 'people.person.gender', 'Female'), ('2013 Billboard Music Awards', 'award.award_ceremony.awards_presented', 'm.0v90p2b'), ('Whitney Houston', 'influence.influence_node.influenced', 'Beyoncé Knowles'), ('Aaliyah', 'people.person.profession', 'Model'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.010htdc2'), ('Eenie Meenie', 'music.single.versions', 'Eenie Meenie'), ('Demi Lovato', 'music.artist.label', 'Island Records'), ('Teen pop', 'common.topic.webpage', 'm.09wmk1t'), ('m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award category'), ('All That Matters', 'music.single.versions', 'All That Matters (video)'), ('All Around The World', 'common.topic.notable_for', 'g.125829xyd'), ('Pattie Mallette', 'common.topic.article', 'm.0h5n_50'), ('Justin Bieber', 'people.person.education', 'm.0w5l5h1'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Katy Perry'), ('Nick Jonas', 'music.artist.genre', 'Teen pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njvs9s'), ('Country of nationality', 'rdf-schema#range', 'Country'), ('m.0101fvps', 'film.film_regional_release_date.film_release_region', 'Italy'), ('Drake', 'music.artist.genre', 'Pop music'), ('Stuart Ford', 'freebase.valuenotation.has_value', 'Country of nationality'), ('Record producer', 'common.topic.subject_of', 'Alan Motley'), ('Rihanna', 'common.topic.notable_types', 'Musical Artist'), ('Live My Life', 'music.album.release_type', 'Single'), ('New Kids on the Block', 'broadcast.artist.content', 'HitzRadio.com'), ('m.0n1ykxp', 'freebase.valuenotation.is_reviewed', 'Ceremony'), ('1Club.FM: Channel One', 'common.topic.notable_for', 'g.1257kpjrj'), ('radioIO Todays RNB', 'broadcast.content.artist', \"Destiny's Child\"), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Three Days Grace'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0_z5ld8'), ('m.0gbm3fj', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Britney Spears', 'broadcast.artist.content', '1.FM Top 40'), ('Brandy Norwood', 'broadcast.artist.content', 'radioIO RNB Mix'), ('Dany Brillant', 'music.artist.genre', 'Pop music'), ('m.0y5t8gm', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrhmll'), ('Lolly', 'music.recording.song', 'Lolly'), ('CSI: Crime Scene Investigation', 'tv.tv_program.languages', 'English Language'), ('1Club.FM: Power', 'broadcast.content.artist', '112'), ('Dr. Dre', 'people.person.profession', 'Artist'), ('m.0pc67ly', 'award.award_nomination.nominated_for', 'Justin Bieber: Never Say Never'), ('m.0vpgdpb', 'music.recording_contribution.contributor', 'Justin Bieber'), ('Jaxon Bieber', 'people.person.parents', 'Jeremy Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.genre', \"00's\"), ('Kylie Minogue', 'music.artist.genre', 'Eurodance'), ('Rodney Jerkins', 'people.person.profession', 'Singer'), ('Wont Stop (feat. Justin Bieber)', 'music.recording.featured_artists', 'Justin Bieber'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Mario'), ('Right Here', 'music.composition.recordings', 'Right Here (featuring Drake)'), ('The Notorious B.I.G.', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('Contemporary R&B', 'music.music_video_genre.music_videos_of_this_genre', 'Josue Diaz'), ('1Club.FM: V101', 'common.topic.image', '1clubfm.jpg'), (\"Justin Bieber's Believe\", 'film.film.genre', 'Documentary film'), ('Nick Jonas', 'people.person.profession', 'Singer'), ('Jennifer Lopez', 'music.artist.genre', 'Dance-pop'), ('m.0101ft0d', 'film.personal_film_appearance.person', 'Mike Posner'), ('Nasri', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long As You Love Me (PAULO & JACKINSKY dub)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Rita Ora', 'music.artist.genre', 'Pop music'), ('As Long As You Love Me (Ferry Corsten club dub)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_721l'), ('m.0ywtfj6', 'award.award_nomination.ceremony', '5th Annual Shorty Awards'), ('Hikaru Utada', 'music.artist.label', 'The Island Def Jam Music Group'), ('Linkin Park', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Green Day'), ('Pattie Mallette', 'people.person.parents', 'Diane Mallette'), ('Everlast', 'common.topic.notable_types', 'Musical Artist'), ('m.0gbmnsr', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'), ('My World', 'music.album.genre', 'Contemporary R&B'), ('My World', 'music.album.releases', 'My World'), ('Tampa', 'common.topic.notable_types', 'City/Town/Village'), ('Beautiful', 'music.recording.tracks', 'Beautiful'), ('Vanessa Hudgens', 'people.person.nationality', 'United States of America'), (\"Kids' Choice Award for Biggest Slime Of The Night\", 'award.award_category.winners', 'm.0sgkw_d'), ('Wait for a Minute', 'common.topic.notable_types', 'Composition'), ('#thatPOWER', 'music.album.release_type', 'Single'), ('Lolly', 'common.topic.notable_types', 'Musical Album'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bl_p'), ('Justin Bieber', 'broadcast.artist.content', 'Sunshine Radio'), ('m.0101ftmz', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('m.0gbm3cp', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Justin Bieber', 'base.popstra.celebrity.dated', 'm.0gxnp0c'), ('Kevin Risto', 'freebase.valuenotation.has_value', 'Date of birth'), ('Eminem', 'music.artist.genre', 'Hip hop music'), ('NME Award for Villain of the Year', 'award.award_category.winners', 'm.0z83x9l'), ('Nicki Minaj', 'music.composer.compositions', 'Beauty And A Beat'), ('Hot Wired Radio', 'broadcast.content.artist', 'Boys Like Girls'), ('Kylie Minogue', 'music.artist.genre', 'Disco'), ('Akon', 'broadcast.artist.content', 'PowerHitz'), ('Amerie', 'music.artist.genre', 'Hip hop music'), ('FLOW 103', 'broadcast.content.artist', 'Beyoncé Knowles'), (\"Justin Bieber's Believe\", 'film.film.costume_design_by', 'Kiyomi Hara'), ('Jay-Z', 'people.person.nationality', 'United States of America'), ('Taylor Swift', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Spouse (or domestic partner)', 'rdf-schema#domain', 'Person'), ('m.0zg2w8r', 'award.award_nomination.nominated_for', 'My World 2.0'), (\"Justin Bieber's Believe\", 'film.film.prequel', 'Justin Bieber: Never Say Never'), ('Justin Bieber', 'people.person.place_of_birth', 'London'), ('Cris Cab', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Everlast', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.producer', 'Beirut Nights Radio'), ('P!nk', 'music.artist.genre', 'Contemporary R&B'), ('Hip hop music', 'broadcast.genre.content', '1Club.FM: Power'), ('Justin Bieber: Just Getting Started', 'common.topic.notable_for', 'g.1259prs0m'), ('As Long as You Love Me', 'music.recording.song', 'As Long as You Love Me'), ('PowerHitz', 'broadcast.content.artist', 'Ludacris'), ('Zac Efron', 'people.person.profession', 'Actor'), ('m.0101fv97', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('L.A. Reid', 'people.person.profession', 'Record producer'), ('m.0zg2qjr', 'award.award_nomination.nominated_for', 'My World 2.0'), ('Kelis', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.featured_artist.recordings', 'Wont Stop (feat. Justin Bieber)'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Linkin Park'), ('Tupac Shakur', 'broadcast.artist.content', '.977 The Hits Channel'), ('Fall Out Boy', 'broadcast.artist.content', 'Hot Wired Radio'), ('Lady Gaga', 'people.person.languages', 'English Language'), ('Deborah Lurie', 'music.artist.genre', 'Pop music'), ('2012 Teen Choice Awards', 'award.award_ceremony.awards_presented', 'm.0yrkgd6'), ('Estelle', 'music.artist.genre', 'Reggae'), ('Wait For a Minute', 'common.topic.notable_types', 'Musical Album'), ('Real Change: Artists for Education', 'common.topic.notable_types', 'Film'), ('Aaliyah', 'music.artist.genre', 'Contemporary R&B'), ('m.0bvmhvb', 'common.webpage.topic', 'Justin Bieber'), ('A Star Was Born: Justin Bieber', 'media_common.netflix_title.netflix_genres', 'Documentary film'), ('Music Producer', 'people.profession.specializations', 'Record producer'), ('Chris Brown', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Kelly Clarkson', 'broadcast.artist.content', '1Club.FM: Channel One'), ('m.0y7y53r', 'award.award_honor.award', 'Kerrang! Villain of the Year Award'), ('radioIO Todays RNB', 'broadcast.content.artist', 'Chrisette Michele'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Fall Out Boy'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vnr'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0y4tdml'), ('Emphatic Radio.com!', 'broadcast.content.artist', 'Paula DeAnda'), ('Next to You', 'music.single.versions', 'Next to You'), ('#thatPOWER', 'music.single.versions', '#thatPOWER'), ('Kelis', 'music.artist.genre', 'Rock music'), ('m.0zbf_4g', 'freebase.valuenotation.has_no_value', 'Winning work'), ('2014 Billboard Music Awards', 'time.event.people_involved', 'Ludacris'), ('The Notorious B.I.G.', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0gj1yzm', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.0h4yhbb', 'film.personal_film_appearance.film', 'A Star Was Born: Justin Bieber'), ('First Dance', 'music.composition.lyricist', 'Justin Bieber'), ('Kanye West', 'broadcast.artist.content', '1Club.FM: Power'), ('Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cw'), ('Contemporary R&B', 'broadcast.genre.content', 'PowerHitz'), ('Never Say Never: The Remixes', 'music.album.releases', 'Never Say Never: The Remixes'), ('Santana', 'broadcast.artist.content', '1Club.FM: V101'), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gv4k'), ('m.0101fs_z', 'film.personal_film_appearance.film', \"Justin Bieber's Believe\"), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Giusy Ferreri'), ('Soulja Boy', 'music.artist.genre', 'Pop music'), ('Nick Jonas', 'people.person.profession', 'Record producer'), ('Believe', 'music.album.album_content_type', 'Studio album'), ('Confident', 'music.recording.artist', 'Justin Bieber'), ('Ginuwine', 'people.person.profession', 'Actor'), ('Chef Tone', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0yrjkl1', 'award.award_honor.award_winner', 'Justin Bieber'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('The Isley Brothers', 'broadcast.artist.content', 'Classic Soul Network'), ('m.0pbzq13', 'film.performance.special_performance_type', 'Uncredited'), ('Next to You', 'music.recording.tracks', 'Next to You'), ('Somebody To Love', 'common.topic.notable_types', 'Musical Recording'), ('m.0115qhzk', 'award.award_honor.ceremony', '2011 MTV Europe Music Awards'), ('Hot Wired Radio', 'broadcast.content.artist', 'Justin Bieber'), ('Book', 'freebase.type_hints.included_types', 'Topic'), ('Pattie Mallette', 'people.person.place_of_birth', 'Stratford'), ('Yves Bole', 'people.person.education', 'm.0129j_53'), ('Juelz Santana', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Brandy Norwood', 'freebase.valuenotation.is_reviewed', 'Official website'), ('First Dance', 'common.topic.image', 'Usher OMG Houston'), ('August Rigo', 'people.person.profession', 'Record producer'), ('Jeremy Bieber', 'people.person.spouse_s', 'm.0101gx29'), ('Kelly Clarkson', 'music.artist.genre', 'Rhythm and blues'), ('All Around The World (featuring Ludacris)', 'music.recording.song', 'All Around The World'), ('Canada', 'base.aareas.schema.administrative_area.administrative_children', 'Ontario'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Pray', 'common.topic.article', 'm.0g5b0p6'), ('Turn to You (Mother’s Day Dedication)', 'music.album.release_type', 'Single'), ('Akon', 'freebase.valuenotation.has_value', 'Children'), ('Die in Your Arms', 'music.album.artist', 'Justin Bieber'), ('Chance the Rapper', 'people.person.nationality', 'United States of America'), ('FLOW 103', 'broadcast.content.genre', 'Urban contemporary'), ('m.0y_g556', 'award.award_honor.honored_for', 'justinbieber'), ('m.0pcr28j', 'tv.tv_guest_role.episodes_appeared_in', 'Targets of Obsession'), ('radioIO Todays POP', 'common.topic.notable_types', 'Broadcast Content'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Beyoncé Knowles'), ('m.0yr9tjb', 'award.award_nomination.award', 'Young Hollywood Award for Newcomer of the Year'), ('Nicki Minaj', 'music.artist.genre', 'Synthpop'), ('Teen Choice Award for Choice Single: Male Artist', 'award.award_category.nominees', 'm.0z84kwx'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Cris Cab', 'people.person.profession', 'Singer-songwriter'), ('Aaliyah', 'people.person.nationality', 'United States of America'), ('K-Ci & JoJo', 'music.artist.genre', 'Rhythm and blues'), ('Maroon 5', 'music.artist.genre', 'Pop music'), ('L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Ronald Isley', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Miley Cyrus', 'people.person.languages', 'English Language'), ('Beauty and a Beat', 'music.recording.canonical_version', 'Beauty and a Beat'), ('Under the Mistletoe', 'award.award_nominated_work.award_nominations', 'm.0z87597'), ('m.0pcr2dh', 'tv.tv_guest_role.episodes_appeared_in', 'Shock Waves'), ('Eminem', 'people.person.profession', 'Musician'), ('m.0z8qx3w', 'award.award_honor.honored_for', 'My World 2.0'), ('1Club.FM: Power', 'broadcast.content.genre', 'Top 40'), ('Justin Bieber', 'common.topic.webpage', 'm.0gc_9w6'), ('Lady Antebellum', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('Miley Cyrus', 'people.person.profession', 'Singer-songwriter'), ('Tupac Shakur', 'people.person.profession', 'Record producer'), ('Michael Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Juvenile', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'), ('Will i Am', 'people.person.nationality', 'United States of America'), ('Terence Dudley', 'music.artist.label', 'The Island Def Jam Music Group'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Terius Nash'), ('Justin Bieber: Never Say Never', 'media_common.netflix_title.netflix_genres', 'Music & Concert Documentaries'), ('Chris Brown', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('#Thatpower', 'music.recording.artist', 'Will i Am'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Rock music'), ('#thatPOWER', 'music.recording.tracks', '#thatPOWER'), ('Scooter Braun', 'music.artist.label', 'RBMG Records'), ('Keyshia Cole', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Britney Spears', 'music.artist.genre', 'Dance-pop'), ('Soulja Boy', 'broadcast.artist.content', 'WildFMRadio.com'), ('Live My Life', 'music.recording.artist', 'Far East Movement'), ('Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'), ('Sean Combs', 'broadcast.artist.content', '.977 The Hits Channel'), ('Justin Bieber', 'music.composer.compositions', 'PYD'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Big R Radio - Top 40 Hits', 'common.topic.article', 'm.03wyslv'), ('m.0w3gbtv', 'film.personal_film_appearance.person', 'Justin Bieber'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Shaffer Smith'), ('Kanye West', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Blu Cantrell'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mims'), ('Lil Wayne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bryan Adams', 'people.person.profession', 'Musician'), ('Ashlee Simpson', 'people.person.nationality', 'United States of America'), ('Hold Tight', 'common.topic.notable_types', 'Musical Recording'), ('Bad Day', 'common.topic.notable_for', 'g.1yl5wl7k0'), ('Sunshine Radio', 'broadcast.content.location', 'Nyíregyháza'), ('Person', 'freebase.type_profile.strict_included_types', 'Topic'), ('Nelly', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Ernie Isley', 'music.artist.genre', 'Rhythm and blues'), ('MTV Video Music Award for Best Male Video', 'award.award_category.winners', 'm.0n4rmg7'), ('Beyoncé Knowles', 'music.artist.genre', 'Contemporary R&B'), ('Jason Mraz', 'people.person.profession', 'Actor'), ('Miami', 'location.location.containedby', 'United States of America'), ('Justin Bieber: Just Getting Started', 'book.book.genre', 'Autobiography'), ('As Long As You Love Me (Audiobot edit)', 'music.recording.artist', 'Justin Bieber'), ('Hot 108 Jamz', 'broadcast.content.artist', 'Sean Paul'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Madonna'), ('Lupe Fiasco', 'people.person.profession', 'Musician'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Third Eye Blind'), ('Anastacia', 'music.artist.genre', 'Rhythm and blues'), ('Musician', 'people.profession.corresponding_type', 'Musical Artist'), ('My World 2.0', 'award.award_nominated_work.award_nominations', 'm.0tjyljn'), ('m.0v_714c', 'film.personal_film_appearance.film', 'Disney Parks Christmas Day Parade'), ('Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Khalil', 'people.person.profession', 'Singer'), ('Selena Gomez', 'base.popstra.business_location.customer', 'm.0gxnp5d'), ('Never Say Never (acoustic)', 'music.recording.song', 'Never Say Never'), ('As Long as You Love Me', 'music.recording.artist', 'Justin Bieber'), ('The Pussycat Dolls', 'broadcast.artist.content', 'radioIO Todays POP'), ('Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Height'), ('Katy Perry: Part of Me', 'film.film.produced_by', 'Katy Perry'), ('m.0gbm3bs', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Tommy Sands', 'base.icons.icon.icon_genre', 'Teen idol'), ('Khalil', 'people.person.nationality', 'United States of America'), ('1.FM Top 40', 'broadcast.content.artist', 'Rihanna'), ('Mary J. Blige', 'people.person.profession', 'Actor'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Frankie J'), ('Beyoncé Knowles', 'people.person.gender', 'Female'), ('Gwen Stefani', 'broadcast.artist.content', 'DeeGay'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('m.012f2m86', 'film.film_film_distributor_relationship.film', \"Justin Bieber's Believe\"), ('Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9k4t'), ('All Around The World', 'music.composition.language', 'English Language'), ('m.0z87597', 'award.award_nomination.nominated_for', 'Under the Mistletoe'), ('m.0101ftnq', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Blu Cantrell', 'music.artist.genre', 'Pop music'), ('m.0ndc3_1', 'award.award_honor.award', 'American Music Award for Artist of the Year'), ('All Around The World', 'music.composition.composer', 'Sir Nolan'), ('Heartbreaker', 'music.composition.recorded_as_album', 'Heartbreaker'), ('Ciara', 'people.person.languages', 'English Language'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvf2'), ('R. Kelly', 'people.person.profession', 'Record producer'), ('Hold Tight', 'common.topic.notable_for', 'g.1yl5pzyrs'), ('Shaffer Smith', 'music.artist.genre', 'Contemporary R&B'), ('Heartbreaker', 'music.album.genre', 'Rhythm and blues'), ('Ray J', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('As Long as You Love Me (acoustic version)', 'common.topic.notable_types', 'Musical Recording'), ('Beauty and a Beat (Bisbetic Instrumental)', 'music.recording.canonical_version', 'Beauty and a Beat'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Mick Quinn'), ('Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Pop music'), ('Shaffer Smith', 'people.person.languages', 'English Language'), ('Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gd28hv'), ('m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Award winner'), ('Usher', 'film.person_or_entity_appearing_in_film.films', 'm.0101fszb'), ('Janet Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'), ('radioIO Todays RNB', 'broadcast.content.genre', 'Rock music'), ('The Isley Brothers', 'music.artist.genre', 'Rock music'), ('Dan Cutforth', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mario'), ('Believe Acoustic', 'music.album.genre', 'Acoustic music'), ('SoulfulSmoothJazz.com', 'common.topic.notable_types', 'Broadcast Content'), ('PowerHitz', 'broadcast.content.artist', 'Beyoncé Knowles'), ('.977 The Hits Channel', 'broadcast.content.artist', \"Plain White T's\"), ('Somebody to Love', 'common.topic.notable_types', 'Musical Album'), ('Electronic dance music', 'music.genre.subgenre', 'Synthpop'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Coldplay'), ('m.0v_73g3', 'tv.tv_guest_personal_appearance.episode', 'Series 7 - Elimination - 8'), ('The Pussycat Dolls', 'music.artist.genre', 'Dance music'), ('Thought of You', 'music.single.versions', 'Thought of You'), ('Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Rita Ora', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Trick Daddy'), ('First Dance', 'common.topic.notable_types', 'Composition'), ('Shaffer Smith', 'broadcast.artist.content', 'Emphatic Radio.com!'), ('Snoop Dogg', 'music.artist.genre', 'Pop music'), ('John Mamann', 'music.composer.compositions', 'Live My Life'), ('New Kids on the Block', 'music.artist.genre', 'Dance music'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njgyk4'), ('Boyfriend', 'music.composition.recordings', 'Boyfriend'), ('Will i Am', 'music.artist.genre', 'Dance music'), ('Enrique Iglesias', 'people.person.profession', 'Musician'), ('Big R Radio - The Hawk', 'broadcast.content.genre', 'Top 40'), ('m.0rqp4h0', 'music.track_contribution.role', 'Vocals'), ('Beauty And A Beat', 'music.composition.recorded_as_album', 'Beauty and a Beat'), ('Kanye West', 'people.person.profession', 'Actor'), ('m.0101fv62', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('1Club.FM: Power', 'broadcast.content.artist', 'Flo Rida'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftqp'), ('Montell Jordan', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Jazmyn Bieber', 'common.topic.notable_types', 'Person'), ('Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0v_72tb'), ('m.0v_98y7', 'tv.tv_guest_personal_appearance.appearance_type', 'Guest host'), ('Vanessa Hudgens', 'music.artist.genre', 'Rhythm and blues'), ('Mary J. Blige', 'people.person.nationality', 'United States of America'), ('Jason Mraz', 'people.person.profession', 'Artist'), ('Nick Jonas', 'freebase.valuenotation.is_reviewed', 'Gender'), ('The Black Eyed Peas', 'common.topic.notable_types', 'Musical Artist'), ('Paul Anka', 'people.person.profession', 'Singer'), ('Somebody to Love', 'music.album.artist', 'Justin Bieber'), ('Marvin Isley', 'music.artist.label', 'The Island Def Jam Music Group'), ('My World 2.0', 'freebase.valuenotation.is_reviewed', 'Artist'), ('Mariah Carey', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'), ('MTV Europe Music Award for Best Pop', 'award.award_category.winners', 'm.0njw59_'), ('m.0yrhmll', 'award.award_honor.award_winner', 'Justin Bieber'), ('Live My Life (Jaywalker remix)', 'music.recording.canonical_version', 'Live My Life'), ('David Cassidy', 'people.person.gender', 'Male'), ('Nicki Minaj', 'music.artist.genre', 'Dance music'), ('Lil Jon', 'people.person.profession', 'Record producer'), ('All Around the World', 'music.recording.featured_artists', 'Ludacris'), ('Never Say Never', 'music.single.versions', 'Never Say Never'), ('Love Never Felt So Good', 'music.composition.composer', 'Paul Anka'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0njhtjj'), ('R. Kelly', 'people.person.place_of_birth', 'Chicago'), ('Tyga', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Big Sean', 'people.person.profession', 'Musician'), ('Kelly Clarkson', 'people.person.profession', 'Actor'), (\"2011 Kids' Choice Awards\", 'award.award_ceremony.awards_presented', 'm.0sgk_cw'), ('justinbieber', 'award.award_nominated_work.award_nominations', 'm.0z2dr9y'), ('Amerie', 'broadcast.artist.content', 'radioIO Todays POP'), ('Beyoncé Knowles', 'broadcast.artist.content', 'radioIO Todays POP'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'The Bronx'), ('Justin Bieber', 'music.composer.compositions', 'Heartbreaker'), ('Jessie J', 'music.artist.genre', 'Dance music'), ('Cory Gunz', 'people.person.gender', 'Male'), ('Bryan Adams', 'people.person.profession', 'Singer-songwriter'), ('Baby (acoustic)', 'music.recording.song', 'Pray'), ('My Worlds Acoustic', 'music.album.artist', 'Justin Bieber'), ('Kylie Minogue', 'music.artist.genre', 'Synthpop'), ('Gwen Stefani', 'music.artist.genre', 'Pop music'), ('Justin Bieber', 'music.featured_artist.recordings', 'Gas Pedal'), ('Beyoncé Knowles', 'music.artist.genre', 'Urban contemporary'), (\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv8j'), ('Eenie Meenie', 'music.composition.recordings', 'Eenie Meenie'), ('m.0d33gyj', 'celebrities.romantic_relationship.relationship_type', 'Dated'), (\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftmm'), ('Jennifer Lopez', 'music.artist.genre', 'Hip hop music'), ('m.0yrlqp1', 'freebase.valuenotation.has_value', 'End Date'), ('As Long as You Love Me', 'music.album.featured_artists', 'Big Sean'), ('#thatPOWER', 'music.recording.tracks', 'ThatPower'), ('Jennifer Lopez', 'people.person.nationality', 'United States of America'), ('Raekwon', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Michael Jackson', 'music.artist.genre', 'Dance-pop'), ('Lolly', 'music.album.featured_artists', 'Juicy J'), ('Sunshine Radio', 'common.topic.notable_for', 'g.125dhx24_'), ('Ciara', 'music.artist.genre', 'Pop music'), ('Usher', 'influence.influence_node.influenced_by', 'Michael Jackson'), ('Singer-songwriter', 'common.topic.notable_types', 'Profession'), ('First Dance', 'music.composition.form', 'Song'), ('Rodney Jerkins', 'music.composer.compositions', 'Roller Coaster'), ('m.0j8mhs_', 'measurement_unit.dated_money_value.currency', 'United States Dollar'), ('Model', 'common.topic.notable_types', 'Profession'), ('1Club.FM: Power', 'broadcast.content.genre', 'Contemporary R&B'), ('Mannie Fresh', 'music.artist.label', 'The Island Def Jam Music Group'), ('All Bad', 'music.composition.composer', 'Dre & Vidal'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Green Day'), ('m.012m1w0t', 'music.group_membership.role', 'Record producer'), ('Ciara', 'broadcast.artist.content', 'Hot Wired Radio'), ('Linkin Park', 'common.topic.notable_types', 'Musical Artist'), ('Redfoo', 'music.artist.genre', 'Synthpop'), ('m.0j8z6tl', 'award.award_nomination.award', 'Billboard Music Award for Top Pop Album'), ('Maroon 5', 'broadcast.artist.content', 'Hot Wired Radio'), ('Runaway Love', 'music.recording.artist', 'Justin Bieber'), ('Paul Anka', 'music.composer.compositions', 'Love Never Felt So Good'), ('Boyfriend', 'music.composition.composer', 'Mike Posner'), ('Justin Bieber', 'music.artist.contribution', 'm.0vpgdpb'), ('Benny Blanco', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('m.0zg8bc1', 'award.award_nomination.ceremony', '2011 Billboard Music Awards'), ('m.0ywvh8k', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0t_l7mp', 'award.award_nomination.ceremony', '2013 Billboard Music Awards'), ('Avery', 'people.person.nationality', 'United States of America'), ('Alicia Keys', 'broadcast.artist.content', '1Club.FM: Channel One'), ('FLOW 103', 'broadcast.content.artist', 'Alicia Keys'), ('Bigger', 'common.topic.article', 'm.0fqlv5q'), ('Toby Gad', 'freebase.valuenotation.has_value', 'Parents'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5th3r'), ('Snoop Dogg', 'broadcast.artist.content', 'HitzRadio.com'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0nh04xx'), ('Shorty Award for Music', 'award.award_category.nominees', 'm.0z0tgz6'), (\"Justin Bieber's Believe\", 'film.film.executive_produced_by', 'Scott Manson'), ('Justin Bieber', 'music.artist.album', 'Beauty and a Beat (Remixes)'), ('Nick Jonas', 'music.artist.label', 'Island Records'), ('My World 2.0', 'music.album.genre', 'Teen pop'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0wjhc6c'), ('#thatPOWER', 'music.recording.tracks', '#ThatPower (feat. Justin Bieber)'), ('Hold Tight', 'music.album.releases', 'Hold Tight'), ('As Long As You Love Me', 'music.single.versions', 'As Long As You Love Me (Ferry Corsten remix)'), ('Eminem', 'broadcast.artist.content', '1Club.FM: Power'), ('DMX', 'people.person.profession', 'Film Producer'), ('Chris Brown', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Justin Bieber', 'music.composer.compositions', 'Bad Day'), ('Yves Bole', 'base.mediaextended.youtube_channel.related_channels', 'Jessie J'), ('As Long As You Love Me (Audien Luvstep mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Victoria Justice', 'people.person.gender', 'Female'), ('Singing', 'people.professional_field.professions_in_this_field', 'Singer-songwriter'), ('Ricky Nelson', 'freebase.valuenotation.is_reviewed', 'Gender'), ('WildFMRadio.com', 'broadcast.content.artist', 'Timbaland'), ('Jessica Simpson', 'people.person.languages', 'English Language'), ('Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Height'), ('Emphatic Radio.com!', 'common.topic.webpage', 'm.046c33q'), ('Alanis Morissette', 'music.artist.genre', 'Pop music'), ('Ja Rule', 'people.person.profession', 'Actor'), ('HitzRadio.com', 'broadcast.content.artist', 'Evanescence'), ('#thatPower', 'music.composition.recordings', '#thatPOWER'), ('A Star Was Born: Justin Bieber', 'common.topic.notable_types', 'Film'), ('Hip hop music', 'broadcast.genre.content', 'Cerritos All Stars Live Mix Show'), ('PYD', 'music.album.featured_artists', 'R. Kelly'), ('Gas Pedal', 'music.recording.artist', 'Justin Bieber'), ('m.011m26pv', 'music.group_membership.group', 'Young Artists for Haiti'), ('Ice Cube', 'common.topic.notable_types', 'Musical Artist'), ('Ciara', 'people.person.profession', 'Singer'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Christina Aguilera'), (\"Destiny's Child\", 'music.artist.genre', 'Urban contemporary'), ('Katy Perry: Part of Me', 'film.film.country', 'United States of America'), ('Demi Lovato', 'music.artist.genre', 'Teen pop'), ('Justin Timberlake', 'freebase.valuenotation.is_reviewed', 'Height'), ('181-beat', 'broadcast.content.genre', 'Urban contemporary'), ('Van Halen', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'), ('#thatPower', 'common.topic.article', 'm.0rytzw4'), ('Jordin Sparks', 'music.artist.genre', 'Rhythm and blues'), ('Ricky Nelson', 'people.person.profession', 'Singer-songwriter'), ('2010 MTV Video Music Awards', 'award.award_ceremony.awards_presented', 'm.0n1ykxp'), ('181-beat', 'broadcast.content.genre', 'Rap music'), ('Contemporary R&B', 'broadcast.genre.content', \"PartyRadioUSA.net - Nation's #1 Party Authority\"), ('Lady Gaga', 'influence.influence_node.influenced_by', 'Whitney Houston'), ('m.0yrk4gn', 'award.award_honor.ceremony', '2012 Teen Choice Awards'), ('Miley Cyrus', 'celebrities.celebrity.celebrity_friends', 'm.012bm2v1'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Harry Chester'), ('Justin Timberlake', 'music.artist.genre', 'Rhythm and blues'), ('New Kids on the Block', 'music.artist.genre', 'Rhythm and blues'), ('Fabolous', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Will Smith', 'people.person.profession', 'Record producer'), ('Lolly', 'common.topic.notable_types', 'Musical Recording'), ('181-beat', 'broadcast.content.artist', 'Trick Daddy'), ('1Club.FM: Power', 'broadcast.content.artist', 'Jennifer Lopez'), ('Rodney Jerkins', 'music.composer.compositions', 'Die in Your Arms'), ('William Orbit', 'people.person.profession', 'Record producer'), ('Jaden Smith', 'music.artist.contribution', 'm.0vp7qr5'), ('m.0v_6_81', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'), ('Montell Jordan', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('iJustine', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('RBMG Records', 'music.record_label.artist', 'Justin Bieber'), ('Gavin DeGraw', 'music.artist.genre', 'Rock music'), ('Amerie', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Rihanna', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Kelis', 'freebase.valuenotation.is_reviewed', 'Gender'), ('m.0pcnqnb', 'film.personal_film_appearance.type_of_appearance', 'Him/Herself'), ('Big R Radio - The Hawk', 'broadcast.content.artist', 'Gavin DeGraw'), ('m.07lkzw7', 'common.webpage.topic', 'Justin Bieber'), ('Blackstreet', 'broadcast.artist.content', 'Big R Radio - The Hawk'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Chrisette Michele'), ('Live My Life', 'music.single.versions', 'Live My Life (Jaywalker remix)'), ('Runaway Love (remix)', 'music.album.album_content_type', 'Remix album'), ('Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Fergie', 'broadcast.artist.content', '1Club.FM: Channel One'), ('Janet Jackson', 'music.artist.label', 'The Island Def Jam Music Group'), ('My World 2.0', 'award.award_winning_work.awards_won', 'm.0njvtth'), ('Fergie', 'people.person.gender', 'Female'), ('Snoop Dogg', 'broadcast.artist.content', 'Hot 108 Jamz'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lil Jon'), ('All Around the World', 'common.topic.notable_types', 'Musical Recording'), ('Live My Life (Party Rock remix)', 'music.recording.song', 'Live My Life'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Eminem'), ('Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Kid Cudi'), ('Believe', 'award.award_nominated_work.award_nominations', 'm.0n58kgb'), ('1Club.FM: V101', 'common.topic.notable_types', 'Broadcast Content'), ('#Thatpower', 'music.recording.artist', 'Justin Bieber'), (\"1Club.FM: Jammin' Oldies\", 'broadcast.content.producer', '1Club.FM'), ('m.0gxnnwp', 'people.sibling_relationship.sibling', 'Justin Bieber'), ('Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Lil Wayne', 'broadcast.artist.content', 'radioIO Todays RNB'), ('Pattie Mallette', 'film.person_or_entity_appearing_in_film.films', 'm.0101fszs'), ('Wait For a Minute', 'music.recording.tracks', 'Wait For a Minute'), ('J. Holiday', 'freebase.valuenotation.has_value', 'Parents'), ('Yves Bole', 'base.fashionmodels.fashion_model.hair_color', 'Brown'), ('SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'), ('Justin Bieber: Never Say Never', 'film.film.production_companies', 'MTV Films'), ('As Long As You Love Me', 'music.single.versions', 'As Long as You Love Me'), ('Alicia Keys', 'music.artist.genre', 'Hip hop music'), ('BeirutNights.com Radio', 'broadcast.content.genre', 'Electronic music'), ('Hot Wired Radio', 'broadcast.content.artist', 'Sean Kingston'), ('Beauty And A Beat', 'award.award_nominated_work.award_nominations', 'm.0_vmmj6'), ('m.0101fv7m', 'film.film_regional_release_date.film', \"Justin Bieber's Believe\"), ('Gavin DeGraw', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('m.0d33hsc', 'celebrities.friendship.friend', 'Selena Gomez'), ('Never Let You Go', 'music.recording.song', 'Never Let You Go'), ('50 Cent', 'broadcast.artist.content', 'PowerHitz'), ('Next to You', 'music.recording.artist', 'Justin Bieber'), ('Sir Mix-a-Lot', 'music.artist.genre', 'Hip hop music'), ('Alicia Keys', 'broadcast.artist.content', 'radioIO Todays POP'), ('Rudolph Valentino', 'common.topic.image', 'Rudolph Valentino'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Avril Lavigne'), ('Sir Mix-a-Lot', 'broadcast.artist.content', '.977 The Hits Channel'), ('Big R Radio - The Hawk', 'broadcast.content.artist', '3 Doors Down'), ('m.0ywtfj6', 'award.award_nomination.award_nominee', 'Justin Bieber'), ('Juicy J', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Victoria Justice', 'people.person.profession', 'Singer-songwriter'), ('Justin Bieber', 'film.person_or_entity_appearing_in_film.films', 'm.0y5t8gm'), ('m.0z8s_wn', 'award.award_honor.award_winner', 'Justin Bieber'), ('Bad Day', 'common.topic.notable_types', 'Musical Album'), ('Electronic dance music', 'common.topic.notable_types', 'Musical genre'), ('radioIO Todays POP', 'broadcast.content.genre', 'Classic hits'), ('The Island Def Jam Music Group', 'music.record_label.artist', 'Cory Gunz'), ('Wait For a Minute', 'common.topic.notable_for', 'g.1yghcn7b9'), ('Justin Bieber', 'music.artist.genre', 'Teen pop'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Mario Winans'), ('m.0ndc3_1', 'award.award_honor.ceremony', 'American Music Awards of 2012'), ('K-Ci & JoJo', 'music.artist.genre', 'Contemporary R&B'), ('m.0y7y53r', 'award.award_honor.award_winner', 'Justin Bieber'), ('m.0yrkr34', 'award.award_honor.award', 'Teen Choice Award for Choice Twitter Personality'), ('1Club.FM: Channel One', 'broadcast.content.artist', 'Kelly Clarkson'), ('Ginuwine', 'music.artist.genre', 'Hip hop music'), ('Next to You', 'music.recording.featured_artists', 'Justin Bieber'), ('Rudolph Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Boyfriend (acoustic version)', 'music.recording.artist', 'Justin Bieber'), ('Max Martin', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Madonna', 'broadcast.artist.content', '1Club.FM: Mix 106'), ('As Long As You Love Me (PAULO & JACKINSKY club mix)', 'music.recording.canonical_version', 'As Long As You Love Me'), ('Will i Am', 'music.artist.album', '#thatPOWER'), ('Never Let You Go', 'common.topic.article', 'm.0bbvzq6'), ('Ashley Tisdale', 'broadcast.artist.content', 'radioIO Todays POP'), ('.977 The Hits Channel', 'broadcast.content.artist', 'Justin Bieber'), ('Artist', 'common.topic.notable_types', 'Profession'), ('Justin Bieber', 'music.featured_artist.recordings', '#thatPOWER'), ('Jeremy Bieber', 'freebase.valuenotation.has_value', 'Parents'), ('Under the Mistletoe', 'music.album.genre', 'Teen pop'), ('Avril Lavigne', 'freebase.valuenotation.is_reviewed', 'Gender'), ('Johnny Crawford', 'people.person.nationality', 'United States of America'), ('m.0101ftn8', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"), ('Spouse', 'rdf-schema#domain', 'Marriage'), ('m.0w5l5h1', 'freebase.valuenotation.has_no_value', 'Degree'), ('Terius Nash', 'freebase.valuenotation.is_reviewed', 'Profession'), ('Trey Songz', 'freebase.valuenotation.is_reviewed', 'Country of nationality'), ('Beauty and a Beat', 'common.topic.notable_types', 'Musical Recording'), ('Wait For a Minute', 'common.topic.notable_for', 'g.1ygjd8zbs'), ('m.0gctps1', 'tv.tv_guest_role.actor', 'Justin Bieber'), ('m.012bq275', 'celebrities.friendship.friend', 'Yves Bole'), ('Montell Jordan', 'people.person.profession', 'Record producer'), ('BeirutNights.com Radio', 'broadcast.content.artist', 'Mars Plastic'), ('Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Date of birth'), ('Confident', 'music.album.release_type', 'Single'), ('Somebody to Love', 'music.composition.composer', 'Justin Bieber'), ('Justin Bieber', 'music.artist.concert_tours', 'Believe Tour'), ('Janet Jackson', 'people.person.languages', 'English Language'), ('Live My Life', 'music.recording.tracks', 'Live My Life'), ('Don Henley', 'people.person.profession', 'Actor'), ('Parents', 'rdf-schema#range', 'Person')]\n" - ] - } - ], - "source": [ - "raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']]\n", - "print(raw_dataset_graphs[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To show the benefits of this indexer in action, we will use the following model to encode this sample of graphs using LargeGraphIndexer, along with naively." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cuda\n" - ] - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)\n", - "print(device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we compare the clock times of encoding using both methods." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [02:01<00:00, 1.22s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "121.99496078491211\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Indexing question-by-question\n", - "dataset_graphs_embedded = []\n", - "start = time.time()\n", - "for graph in tqdm.tqdm(raw_dataset_graphs):\n", - " nodes_map = dict()\n", - " edges_map = dict()\n", - " edge_idx_base = []\n", - "\n", - " for src, edge, dst in graph:\n", - " # Collect nodes\n", - " if src not in nodes_map:\n", - " nodes_map[src] = len(nodes_map)\n", - " if dst not in nodes_map:\n", - " nodes_map[dst] = len(nodes_map)\n", - " \n", - " # Collect edge types\n", - " if edge not in edges_map:\n", - " edges_map[edge] = len(edges_map)\n", - "\n", - " # Record edge\n", - " edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst]))\n", - " \n", - " # Encode nodes and edges\n", - " sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x]))\n", - " sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x]))\n", - "\n", - " x = model.encode(sorted_nodes, batch_size=256)\n", - " edge_attrs_map = model.encode(sorted_edges, batch_size=256)\n", - " \n", - " edge_attrs = []\n", - " edge_idx = []\n", - " for trip in edge_idx_base:\n", - " edge_attrs.append(edge_attrs_map[trip[1]])\n", - " edge_idx.append([trip[0], trip[2]])\n", - "\n", - " dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0)))\n", - " \n", - " \n", - "print(time.time()-start)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Indexing...\n", - "Time to create whole knowledge_graph: 114.14257955551147\n", - "Retrieving Subgraphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [00:00<00:00, 206.96it/s]\n", - "100%|██████████| 100/100 [00:01<00:00, 83.32it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "114.77728700637817\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Using LargeGraphIndexer to make one large knowledge graph\n", - "from torch_geometric.data.large_graph_indexer import EDGE_RELATION\n", - "\n", - "start = time.time()\n", - "all_triplets_together = chain.from_iterable(raw_dataset_graphs)\n", - "# Index as one large graph\n", - "print('Indexing...')\n", - "indexer = LargeGraphIndexer.from_triplets(all_triplets_together)\n", - "\n", - "# first the nodes\n", - "unique_nodes = indexer.get_unique_node_features()\n", - "node_encs = model.encode(unique_nodes, batch_size=256)\n", - "indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs)\n", - "\n", - "# then the edges\n", - "unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION)\n", - "edge_attr = model.encode(unique_edges, batch_size=256)\n", - "indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION)\n", - "\n", - "ckpt_time = time.time()\n", - "whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') \n", - "whole_graph_done = time.time()\n", - "print(f\"Time to create whole knowledge_graph: {whole_graph_done-start}\")\n", - "\n", - "# Compute this to make sure we're comparing like to like on final time printout\n", - "whole_graph_diff = whole_graph_done-ckpt_time\n", - "\n", - "# retrieve subgraphs\n", - "print('Retrieving Subgraphs...')\n", - "dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)]\n", - "print(time.time()-start-whole_graph_diff)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The large graph indexer allows us to compute the entire knowledge graph from a series of samples, so that new retrieval methods can also be tested on the entire graph. We will see this attempted in practice later on." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's worth noting that, although the times are relatively similar right now, the speedup with largegraphindexer will be much higher as the size of the knowledge graph grows. This is due to the speedup being a factor of the number of unique nodes and edges in the graph." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]),\n", - " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]),\n", - " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]),\n", - " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]),\n", - " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]),\n", - " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]),\n", - " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]),\n", - " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]),\n", - " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]),\n", - " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]),\n", - " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]),\n", - " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]),\n", - " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]),\n", - " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]),\n", - " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]),\n", - " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]),\n", - " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", - " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]),\n", - " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]),\n", - " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]),\n", - " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]),\n", - " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]),\n", - " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]),\n", - " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]),\n", - " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]),\n", - " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]),\n", - " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]),\n", - " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]),\n", - " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]),\n", - " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]),\n", - " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]),\n", - " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]),\n", - " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]),\n", - " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]),\n", - " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]),\n", - " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", - " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]),\n", - " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]),\n", - " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]),\n", - " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]),\n", - " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]),\n", - " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]),\n", - " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]),\n", - " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]),\n", - " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]),\n", - " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]),\n", - " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]),\n", - " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]),\n", - " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]),\n", - " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]),\n", - " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]),\n", - " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]),\n", - " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]),\n", - " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]),\n", - " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]),\n", - " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]),\n", - " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]),\n", - " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]),\n", - " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]),\n", - " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]),\n", - " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]),\n", - " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]),\n", - " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]),\n", - " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]),\n", - " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]),\n", - " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]),\n", - " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]),\n", - " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]),\n", - " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]),\n", - " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]),\n", - " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", - " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", - " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]),\n", - " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]),\n", - " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]),\n", - " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]),\n", - " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]),\n", - " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]),\n", - " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]),\n", - " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]),\n", - " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]),\n", - " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]),\n", - " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]),\n", - " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]),\n", - " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", - " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", - " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]),\n", - " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset_graphs_embedded_largegraphindexer" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]),\n", - " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]),\n", - " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]),\n", - " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]),\n", - " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]),\n", - " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]),\n", - " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]),\n", - " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]),\n", - " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]),\n", - " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]),\n", - " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]),\n", - " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]),\n", - " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]),\n", - " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]),\n", - " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]),\n", - " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]),\n", - " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", - " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]),\n", - " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]),\n", - " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]),\n", - " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]),\n", - " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]),\n", - " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]),\n", - " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]),\n", - " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]),\n", - " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]),\n", - " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]),\n", - " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]),\n", - " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]),\n", - " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]),\n", - " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]),\n", - " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]),\n", - " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]),\n", - " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]),\n", - " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]),\n", - " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", - " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]),\n", - " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]),\n", - " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]),\n", - " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]),\n", - " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]),\n", - " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]),\n", - " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]),\n", - " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]),\n", - " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]),\n", - " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]),\n", - " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]),\n", - " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]),\n", - " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]),\n", - " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]),\n", - " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]),\n", - " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]),\n", - " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]),\n", - " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]),\n", - " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]),\n", - " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]),\n", - " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]),\n", - " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]),\n", - " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]),\n", - " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]),\n", - " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]),\n", - " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]),\n", - " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]),\n", - " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]),\n", - " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]),\n", - " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]),\n", - " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]),\n", - " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", - " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", - " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]),\n", - " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]),\n", - " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]),\n", - " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]),\n", - " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]),\n", - " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]),\n", - " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]),\n", - " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]),\n", - " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]),\n", - " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]),\n", - " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]),\n", - " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]),\n", - " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", - " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", - " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]),\n", - " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset_graphs_embedded" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We expect the two results to be functionally identical, with the differences being due to floating point jitter." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", - " def _sorted_tensors_are_close(tensor1, tensor2):\n", - " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", - " def _graphs_are_same(tensor1, tensor2):\n", - " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", - " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", - " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", - " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/100 [00:00 2\u001b[0m ds_updated \u001b[38;5;241m=\u001b[39m \u001b[43mUpdatedWebQSPDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mupdated_dataset\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(time\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;241m-\u001b[39mstart)\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:182\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.__init__\u001b[0;34m(self, root, force_reload, whole_graph_retrieval, limit, override)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moverride \u001b[38;5;241m=\u001b[39m override\n\u001b[1;32m 180\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mdevice(\n\u001b[1;32m 181\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 182\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_raw_data()\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m0\u001b[39m])\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/in_memory_dataset.py:81\u001b[0m, in \u001b[0;36mInMemoryDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 74\u001b[0m root: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m force_reload: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 80\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mroot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_transform\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpre_filter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 82\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_reload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data: Optional[BaseData] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslices: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:115\u001b[0m, in \u001b[0;36mDataset.__init__\u001b[0;34m(self, root, transform, pre_transform, pre_filter, log, force_reload)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download()\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhas_process:\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/data/dataset.py:260\u001b[0m, in \u001b[0;36mDataset._process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcessing...\u001b[39m\u001b[38;5;124m'\u001b[39m, file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstderr)\n\u001b[1;32m 259\u001b[0m fs\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 262\u001b[0m path \u001b[38;5;241m=\u001b[39m osp\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpre_transform.pt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 263\u001b[0m fs\u001b[38;5;241m.\u001b[39mtorch_save(_repr(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_transform), path)\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:328\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset.process\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_paths[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]):\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncoding graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 328\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 330\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading graph...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/datasets/updated_web_qsp_dataset.py:260\u001b[0m, in \u001b[0;36mUpdatedWebQSPDataset._build_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;66;03m# Nodes:\u001b[39;00m\n\u001b[1;32m 259\u001b[0m nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindexer\u001b[38;5;241m.\u001b[39mget_unique_node_features()\n\u001b[0;32m--> 260\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindexer\u001b[38;5;241m.\u001b[39madd_node_feature(new_feature_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mx\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_feature_vals\u001b[38;5;241m=\u001b[39mx)\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# Edges:\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/torch_geometric/nn/nlp/sentence_transformer.py:75\u001b[0m, in \u001b[0;36mSentenceTransformer.encode\u001b[0;34m(self, text, batch_size, output_device)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m start \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(text), batch_size):\n\u001b[1;32m 65\u001b[0m token \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtokenizer(\n\u001b[1;32m 66\u001b[0m text[start:start \u001b[38;5;241m+\u001b[39m batch_size],\n\u001b[1;32m 67\u001b[0m padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 68\u001b[0m truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 69\u001b[0m return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 70\u001b[0m )\n\u001b[1;32m 72\u001b[0m emb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m 73\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 74\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m---> 75\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_device\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcpu\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 77\u001b[0m embs\u001b[38;5;241m.\u001b[39mappend(emb)\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat(embs, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(embs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m embs[\u001b[38;5;241m0\u001b[39m]\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "start = time.time()\n", - "ds_updated = UpdatedWebQSPDataset('updated_dataset', limit=demo_number_of_graphs)\n", - "updated_time = time.time() - start\n", - "print(time.time()-start)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "plt.bar(['WebQSP', 'UpdatedQSP'], [web_qsp_time,, updated_time])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example, we will take a set of multi-hop questions, and combine them with an existing Wikidata knowledge graph to produce a new dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To be continued in 0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/doc/media/multihop_example.svg b/examples/llm_plus_gnn/doc/media/multihop_example.svg new file mode 100644 index 000000000000..a6cb75681b48 --- /dev/null +++ b/examples/llm_plus_gnn/doc/media/multihop_example.svg @@ -0,0 +1 @@ + \ No newline at end of file From 77cd9a761c5f244cb38d566f267421a5076f8b83 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:15:20 -0700 Subject: [PATCH 648/752] Prune out everything that doesn't have to do with LargeGraphIndexer --- .../llm_plus_gnn/benchmark_model_archs.py | 37 - .../llm_plus_gnn/benchmark_model_gnn_enc.py | 39 - examples/llm_plus_gnn/g_retriever.py | 224 +- examples/llm_plus_gnn/nvtx_run.sh | 27 - examples/llm_plus_gnn/profiling_utils.py | 193 - examples/llm_plus_gnn/rag_feature_store.py | 93 - examples/llm_plus_gnn/rag_generate.py | 75 - .../llm_plus_gnn/rag_generate_multihop.py | 38 - examples/llm_plus_gnn/rag_graph_store.py | 92 - examples/llm_plus_gnn/raw_qsp_dataset.py | 139 - .../test_eval_large_graph_indexer.ipynb | 737 - .../test_eval_updated_webqsp.ipynb | 559 - examples/llm_plus_gnn/test_g_retriever.ipynb | 363 - .../llm_plus_gnn/test_nvtx_rag_backend.py | 101 - examples/llm_plus_gnn/test_nvtx_uwebqsp.py | 27 - examples/llm_plus_gnn/test_nvtx_webqsp.py | 19 - examples/llm_plus_gnn/test_profiling.ipynb | 2707 -- examples/llm_plus_gnn/test_rag.ipynb | 321 - examples/llm_plus_gnn/wikidata.ipynb | 30776 ---------------- torch_geometric/datasets/web_qsp_dataset.py | 21 +- torch_geometric/loader/__init__.py | 2 - torch_geometric/loader/rag_loader.py | 88 - torch_geometric/nn/models/g_retriever.py | 5 +- torch_geometric/profile/__init__.py | 2 - torch_geometric/profile/nvtx.py | 55 - torch_geometric/profile/profiler.py | 8 +- 26 files changed, 51 insertions(+), 36697 deletions(-) delete mode 100644 examples/llm_plus_gnn/benchmark_model_archs.py delete mode 100644 examples/llm_plus_gnn/benchmark_model_gnn_enc.py delete mode 100755 examples/llm_plus_gnn/nvtx_run.sh delete mode 100644 examples/llm_plus_gnn/profiling_utils.py delete mode 100644 examples/llm_plus_gnn/rag_feature_store.py delete mode 100644 examples/llm_plus_gnn/rag_generate.py delete mode 100644 examples/llm_plus_gnn/rag_generate_multihop.py delete mode 100644 examples/llm_plus_gnn/rag_graph_store.py delete mode 100644 examples/llm_plus_gnn/raw_qsp_dataset.py delete mode 100644 examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb delete mode 100644 examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb delete mode 100644 examples/llm_plus_gnn/test_g_retriever.ipynb delete mode 100644 examples/llm_plus_gnn/test_nvtx_rag_backend.py delete mode 100644 examples/llm_plus_gnn/test_nvtx_uwebqsp.py delete mode 100644 examples/llm_plus_gnn/test_nvtx_webqsp.py delete mode 100644 examples/llm_plus_gnn/test_profiling.ipynb delete mode 100644 examples/llm_plus_gnn/test_rag.ipynb delete mode 100644 examples/llm_plus_gnn/wikidata.ipynb delete mode 100644 torch_geometric/loader/rag_loader.py delete mode 100644 torch_geometric/profile/nvtx.py diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py deleted file mode 100644 index f73d16b7ed9d..000000000000 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ /dev/null @@ -1,37 +0,0 @@ -# %% -from g_retriever import benchmark_models, get_loss, inference_step -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.models import GRetriever, MLP, GAT - -# %% -hidden_channels = 1024 -num_gnn_layers = 1 -lr=1e-5 -epochs=2 -batch_size=8 -eval_batch_size=16 - -# %% -ds = UpdatedWebQSPDataset('benchmark_archs') - -# %% -model_names = [] -model_classes = [] -model_kwargs = [] -model_type = ["GAT", "MLP"] -models = {"GAT": GAT, "MLP": MLP} -num_layers = [1, 2, 4, 8, 16, 32] -for m_type in model_type: - for n_layer in num_layers: - model_names.append(f"{m_type}_{n_layer}") - model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels, num_gnn_layers=n_layer, gnn_to_use=models[m_type]) - model_kwargs.append(kwargs) - -# %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) - -# %% - - - diff --git a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py deleted file mode 100644 index db1210b70ed4..000000000000 --- a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py +++ /dev/null @@ -1,39 +0,0 @@ -# %% -from g_retriever import benchmark_models, get_loss, inference_step -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.models import GRetriever, MLP, GAT - -# %% -hidden_channels = 1024 -num_gnn_layers = 1 -lr=1e-5 -epochs=2 -batch_size=8 -eval_batch_size=16 - -# %% -ds = UpdatedWebQSPDataset('benchmark_archs') - -# %% -model_names = [] -model_classes = [] -model_kwargs = [] -model_type = ["GAT"] -models = {"GAT": GAT, "MLP": MLP} -num_layers = [4] -num_tokens = [4, 16] -for m_type in model_type: - for n_layer in num_layers: - for n_tokens in num_tokens - model_names.append(f"{m_type}_{n_layer}_{n_tokens}") - model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels*n_tokens, num_gnn_layers=n_layer, gnn_to_use=models[m_type], gnn_out_channels=hidden_channels*n_tokens, mlp_out_tokens=n_tokens) - model_kwargs.append(kwargs) - -# %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) - -# %% - - - diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1d0956ed4840..a6af4afb3eaf 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -16,19 +16,15 @@ import torch from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm -import torch.nn as nn -from typing import Callable, List, Type, Dict, Any from torch_geometric import seed_everything -from torch_geometric.data import Dataset -from torch_geometric.datasets import UpdatedWebQSPDataset, WebQSPDataset +from torch_geometric.datasets import WebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM -import multiprocessing as mp -def _detect_hallucinate(inp): - pred, label = inp + +def detect_hallucinate(pred, label): try: split_pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(split_pred[0], label)) > 0 @@ -92,9 +88,6 @@ def compute_accuracy(eval_output) -> float: return hit -def compute_n_parameters(model: torch.nn.Module) -> int: - return sum([p.numel() for p in model.parameters() if p.requires_grad]) - def save_params_dict(model, save_path): state_dict = model.state_dict() @@ -132,10 +125,9 @@ def inference_step(model, batch, model_save_name): batch.desc) - def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=None, dataset=None, - checkpointing=False, tiny_llama=False, model_save_name=None): + checkpointing=False, tiny_llama=False): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -162,7 +154,6 @@ def adjust_learning_rate(param_group, LR, epoch): train_dataset = [dataset[i] for i in idx_split['train']] val_dataset = [dataset[i] for i in idx_split['val']] test_dataset = [dataset[i] for i in idx_split['test']] - print(len(train_dataset), len(val_dataset), len(test_dataset)) train_loader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True, pin_memory=True, shuffle=True) @@ -186,11 +177,10 @@ def adjust_learning_rate(param_group, LR, epoch): model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) - if model_save_name is None: - if num_gnn_layers is not None: - model_save_name = "gnn_llm" - else: - model_save_name = "llm" + if num_gnn_layers is not None: + model_save_name = "gnn_llm" + else: + model_save_name = "llm" # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -283,138 +273,6 @@ def adjust_learning_rate(param_group, LR, epoch): print("Done!") return prep_time, dataset, eval_output -def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): - model_save_list = [] - model_preds = [] - for out in outs: - model_preds += out['pred'] - for i, batch in enumerate(loader): - correct_answer = batch.label - - model_pred = model_preds[i*eval_batch_size:(i+1)*eval_batch_size] - model_hallucinates = detect_hallucinate(model_pred, - correct_answer) - model_save_list += [tup for tup in zip(model_pred, model_hallucinates)] - return model_save_list - - -def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True, force: bool = False, root_dir='.'): - - model_log: Dict[str, Dict[str, Any]] = dict() - idx_split = dataset.split_idxs - test_dataset = [dataset[i] for i in idx_split['test']] - loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, - pin_memory=True, shuffle=False) - - if not skip_LLMs: - if tiny_llama: - pure_llm = LLM( - model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - num_params=1, - ) - else: - pure_llm = LLM(model_name="llama2-7b", num_params=7) - - if not path.exists(root_dir +"/pure_llm_model_log.pt"): - model_log["pure_llm"] = dict() - - pure_preds = [] - for batch in tqdm(loader): - pure_llm_preds = pure_llm.inference(batch.question, batch.desc, max_tokens=256) - pure_preds += pure_llm_preds - pure_preds = [{"pred": pred} for pred in pure_preds] - - model_log["pure_llm"]["preds"] = pure_preds - model_log["pure_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(pure_preds, loader, eval_batch_size) - model_log["pure_llm"]["n_params"] = compute_n_parameters(pure_llm) - torch.save(model_log["pure_llm"], root_dir+"/pure_llm_model_log.pt") - else: - model_log["pure_llm"] = torch.load(root_dir+"/pure_llm_model_log.pt") - - # LORA - if not path.exists(root_dir+"/tuned_llm_model_log.pt"): - model_log["tuned_llm"] = dict() - since = time.time() - gc.collect() - prep_time, _, lora_eval_outs = train(since, epochs, None, None, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=pure_llm, dataset=dataset) - torch.cuda.empty_cache() - torch.cuda.reset_max_memory_allocated() - gc.collect() - e2e_time = round(time.time() - since, 2) - model_log["tuned_llm"]["prep_time"] = prep_time - model_log["tuned_llm"]["e2e_time"] = e2e_time - model_log["tuned_llm"]["eval_output"] = lora_eval_outs - print("E2E time (e2e_time) =", e2e_time, "seconds") - print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") - - model_log["tuned_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(lora_eval_outs, loader, eval_batch_size) - model_log["tuned_llm"]["n_params"] = compute_n_parameters(pure_llm) - torch.save(model_log["tuned_llm"], root_dir+"/tuned_llm_model_log.pt") - else: - model_log["tuned_llm"] = torch.load(root_dir+"/tuned_llm_model_log.pt") - - del pure_llm - gc.collect() - - # All other models - for name, Model, kwargs in zip(model_names, models, model_kwargs): - model_log[name] = dict() - train_model = True - if path.exists(root_dir+f"/{name}.pt") and not force: - print(f"Model {name} appears to already exist.") - print("Would you like to retrain?") - train_model = str(input("(y/n):")).lower() == "y" - - if train_model: - since = time.time() - gc.collect() - model = Model(**kwargs) - prep_time, _, model_eval_outs = train( - since=since, num_epochs=epochs, hidden_channels=None, num_gnn_layers=None, - batch_size=batch_size, eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, - inference_fn=inference_fn, checkpointing=checkpointing, - tiny_llama=tiny_llama, dataset=dataset, model_save_name=root_dir+'/'+name, model=model) - torch.cuda.empty_cache() - torch.cuda.reset_max_memory_allocated() - gc.collect() - e2e_time = round(time.time() - since, 2) - model_log[name]["prep_time"] = prep_time - model_log[name]["e2e_time"] = e2e_time - model_log[name]["eval_output"] = model_eval_outs - print("E2E time (e2e_time) =", e2e_time, "seconds") - print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") - model_log[name]["n_params"] = compute_n_parameters(model) - del model - gc.collect() - else: - model_eval_outs = torch.load(root_dir+f"/{name}_eval_outs.pt") - - # Calculate Hallucinations - skip_hallucination_detection = False - - if path.exists(root_dir+f"/{name}_model_log.pt") and not force: - print(f"Saved outputs for {name} have been found.") - print("Would you like to redo?") - user_input = str(input("(y/n):")).lower() - skip_hallucination_detection = user_input != "y" - - - if not skip_hallucination_detection: - model_save_list = _eval_hallucinations_on_loader(model_eval_outs, loader, eval_batch_size) - - model_log[name]["hallucinates_list"] = model_save_list - torch.save(model_log[name], root_dir+f"/{name}_model_log.pt") - else: - model_log[name]["hallucinates_list"] = torch.load(root_dir+f"/{name}_model_log.pt")["hallucinates_list"] - - hal_dict = {k: [tup[1] for tup in v["hallucinates_list"]] for (k, v) in model_log.items()} - hallucinates_df = pd.DataFrame(hal_dict).astype(str) - hallucinates_df = hallucinates_df.apply(pd.Series.value_counts).transpose() - hallucinates_df['e2e_time'] = pd.Series({k: v.get('e2e_time') for (k, v) in model_log.items()}) - hallucinates_df['n_params'] = pd.Series({k: v.get('n_params') for (k, v) in model_log.items()}) - print(hallucinates_df) - hallucinates_df.to_csv(root_dir+"/hallucinates_df.csv", index=False) - def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size, loss_fn, inference_fn, @@ -425,7 +283,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, + loader = DataLoader(test_dataset, batch_size=1, drop_last=False, pin_memory=True, shuffle=False) if tiny_llama: pure_llm = LLM( @@ -458,8 +316,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa ) for i, batch in enumerate(tqdm(loader)): - question = batch.question - correct_answer = batch.label + question = batch.question[0] + correct_answer = batch.label[0] if skip_pretrained_LLM: pure_llm_pred = None pure_llm_hallucinates = False @@ -470,19 +328,18 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [tup for tup in zip(pure_llm_pred, pure_llm_hallucinates)] + untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] - gnn_llm_pred = gnn_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] + gnn_llm_pred = gnn_llm_preds[i] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates)] - - for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, pure_llm_hallucinates): - if gnn_llm_hal == "skip" or pure_llm_hal == "skip": # noqa - # skipping when hallucination is hard to eval - continue - gnn_llm_hallucin_sum += int(gnn_llm_hal) - pure_llm_hallucin_sum += int(pure_llm_hal) + gnn_save_list += [(gnn_llm_pred, gnn_llm_hallucinates)] + + if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa + # skipping when hallucination is hard to eval + continue + gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) + pure_llm_hallucin_sum += int(pure_llm_hallucinates) if not skip_pretrained_LLM: print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) @@ -533,31 +390,30 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_preds += out['pred'] print("Final comparison between all models...") for i, batch in enumerate(tqdm(loader)): - question = batch.question - correct_answer = batch.label - gnn_llm_pred, gnn_llm_hallucinates = list(zip(*gnn_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) - untuned_llm_pred, untuned_llm_hallucinates = list(zip(*untuned_llm_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) + question = batch.question[0] + correct_answer = batch.label[0] + gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] + untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] + pure_llm_pred = pure_llm_preds[i] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) - for j in range(len(gnn_llm_pred)): - if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[j] == "skip" or pure_llm_hallucinates[j] == "skip": - continue - trained_llm_hallucin_sum += int(pure_llm_hallucinates[j]) - if skip_pretrained_LLM: - # we did not check the untrained LLM, so do not decide to demo - # based on this. - untuned_llm_hallucinates = True - if untuned_llm_hallucinates[j] and pure_llm_hallucinates[j] and not gnn_llm_hallucinates[j]: # noqa - final_prnt_str += "Prompt: '" + question[j] + "'\n" - final_prnt_str += "Label: '" + correct_answer[j] + "'\n" - if not skip_pretrained_LLM: - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[j] + "'\n" # noqa - final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[j] + "'\n" - final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred[j] + "'\n" - final_prnt_str += "\n" + "#" * 20 + "\n\n" + if pure_llm_hallucinates == "skip": + continue + trained_llm_hallucin_sum += int(pure_llm_hallucinates) + if skip_pretrained_LLM: + # we did not check the untrained LLM, so do not decide to demo + # based on this. + untuned_llm_hallucinates = True + if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa + final_prnt_str += "Prompt: '" + question + "'\n" + final_prnt_str += "Label: '" + correct_answer + "'\n" + if not skip_pretrained_LLM: + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa + final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" + final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" + final_prnt_str += "\n" + "#" * 20 + "\n\n" if not skip_pretrained_LLM: print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_run.sh deleted file mode 100755 index b40c32abb0ea..000000000000 --- a/examples/llm_plus_gnn/nvtx_run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# Check if the user provided a Python file -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -# Check if the provided file exists -if [[ ! -f "$1" ]]; then - echo "Error: File '$1' does not exist." - exit 1 -fi - -# Check if the provided file is a Python file -if [[ ! "$1" == *.py ]]; then - echo "Error: '$1' is not a Python file." - exit 1 -fi - -# Get the base name of the Python file -python_file=$(basename "$1") - -# Run nsys profile on the Python file -nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cuda-memory-usage true --cudabacktrace all --force-overwrite true --output=profile_${python_file%.py} python "$1" - -echo "Profile data saved as profile_${python_file%.py}.nsys-rep" \ No newline at end of file diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py deleted file mode 100644 index 2bc537f08d4a..000000000000 --- a/examples/llm_plus_gnn/profiling_utils.py +++ /dev/null @@ -1,193 +0,0 @@ -from dataclasses import dataclass -from enum import Enum, auto -from typing import ( - Any, - Callable, - Dict, - Iterable, - Optional, - Protocol, - Tuple, - Type, - runtime_checkable, -) - -import torch -from torch import Tensor -from torch.nn import Module - -from torch_geometric.data import ( - FeatureStore, - GraphStore, - LargeGraphIndexer, - TripletLike, -) -from torch_geometric.data.large_graph_indexer import EDGE_RELATION -from torch_geometric.distributed import ( - LocalFeatureStore, - LocalGraphStore, - Partitioner, -) -from torch_geometric.typing import EdgeType, NodeType - -RemoteGraphBackend = Tuple[FeatureStore, GraphStore] - -# TODO: Make everything compatible with Hetero graphs aswell - - -# Adapted from LocalGraphStore -@runtime_checkable -class ConvertableGraphStore(Protocol): - @classmethod - def from_data( - cls, - edge_id: Tensor, - edge_index: Tensor, - num_nodes: int, - is_sorted: bool = False, - ) -> GraphStore: - ... - - @classmethod - def from_hetero_data( - cls, - edge_id_dict: Dict[EdgeType, Tensor], - edge_index_dict: Dict[EdgeType, Tensor], - num_nodes_dict: Dict[NodeType, int], - is_sorted: bool = False, - ) -> GraphStore: - ... - - @classmethod - def from_partition(cls, root: str, pid: int) -> GraphStore: - ... - - -# Adapted from LocalFeatureStore -@runtime_checkable -class ConvertableFeatureStore(Protocol): - @classmethod - def from_data( - cls, - node_id: Tensor, - x: Optional[Tensor] = None, - y: Optional[Tensor] = None, - edge_id: Optional[Tensor] = None, - edge_attr: Optional[Tensor] = None, - ) -> FeatureStore: - ... - - @classmethod - def from_hetero_data( - cls, - node_id_dict: Dict[NodeType, Tensor], - x_dict: Optional[Dict[NodeType, Tensor]] = None, - y_dict: Optional[Dict[NodeType, Tensor]] = None, - edge_id_dict: Optional[Dict[EdgeType, Tensor]] = None, - edge_attr_dict: Optional[Dict[EdgeType, Tensor]] = None, - ) -> FeatureStore: - ... - - @classmethod - def from_partition(cls, root: str, pid: int) -> FeatureStore: - ... - - -class RemoteDataType(Enum): - DATA = auto() - PARTITION = auto() - - -@dataclass -class RemoteGraphBackendLoader: - path: str - datatype: RemoteDataType - graph_store_type: Type[ConvertableGraphStore] - feature_store_type: Type[ConvertableFeatureStore] - - def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: - if self.datatype == RemoteDataType.DATA: - data_obj = torch.load(self.path) - graph_store = self.graph_store_type.from_data( - edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, - num_nodes=data_obj.num_nodes) - feature_store = self.feature_store_type.from_data( - node_id=data_obj['node_id'], x=data_obj.x, - edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) - elif self.datatype == RemoteDataType.PARTITION: - if pid is None: - assert pid is not None, \ - "Partition ID must be defined for loading from a " \ - + "partitioned store." - graph_store = self.graph_store_type.from_partition(self.path, pid) - feature_store = self.feature_store_type.from_partition( - self.path, pid) - else: - raise NotImplementedError - return (feature_store, graph_store) - - -# TODO: make profilable -def create_remote_backend_from_triplets( - triplets: Iterable[TripletLike], node_embedding_model: Module, - edge_embedding_model: Module | None = None, - graph_db: Type[ConvertableGraphStore] = LocalGraphStore, - feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, - node_method_to_call: str = "forward", - edge_method_to_call: str | None = None, - pre_transform: Callable[[TripletLike], TripletLike] | None = None, - path: str = '', n_parts: int = 1, - node_method_kwargs: Optional[Dict[str, Any]] = None, - edge_method_kwargs: Optional[Dict[str, Any]] = None -) -> RemoteGraphBackendLoader: - - # Will return attribute errors for missing attributes - if not issubclass(graph_db, ConvertableGraphStore): - getattr(graph_db, "from_data") - getattr(graph_db, "from_hetero_data") - getattr(graph_db, "from_partition") - elif not issubclass(feature_db, ConvertableFeatureStore): - getattr(feature_db, "from_data") - getattr(feature_db, "from_hetero_data") - getattr(feature_db, "from_partition") - - # Resolve callable methods - node_method_kwargs = node_method_kwargs \ - if node_method_kwargs is not None else dict() - - edge_embedding_model = edge_embedding_model \ - if edge_embedding_model is not None else node_embedding_model - edge_method_to_call = edge_method_to_call \ - if edge_method_to_call is not None else node_method_to_call - edge_method_kwargs = edge_method_kwargs \ - if edge_method_kwargs is not None else node_method_kwargs - - # These will return AttributeErrors if they don't exist - node_model = getattr(node_embedding_model, node_method_to_call) - edge_model = getattr(edge_embedding_model, edge_method_to_call) - - indexer = LargeGraphIndexer.from_triplets(triplets, - pre_transform=pre_transform) - - node_feats = node_model(indexer.get_node_features(), **node_method_kwargs) - indexer.add_node_feature('x', node_feats) - - edge_feats = edge_model( - indexer.get_unique_edge_features(feature_name=EDGE_RELATION), - **edge_method_kwargs) - indexer.add_edge_feature(new_feature_name="edge_attr", - new_feature_vals=edge_feats, - map_from_feature=EDGE_RELATION) - - data = indexer.to_data(node_feature_name='x', - edge_feature_name='edge_attr') - - if n_parts == 1: - torch.save(data, path) - return RemoteGraphBackendLoader(path, RemoteDataType.DATA, graph_db, - feature_db) - else: - partitioner = Partitioner(data=data, num_parts=n_parts, root=path) - partitioner.generate_partition() - return RemoteGraphBackendLoader(path, RemoteDataType.PARTITION, - graph_db, feature_db) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py deleted file mode 100644 index ca3a0ba646d4..000000000000 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ /dev/null @@ -1,93 +0,0 @@ -from collections.abc import Iterable, Iterator -from typing import Any, Dict, Optional, Type, Union - -import torch -from torch import Tensor -from torch.nn import Module -from torchmetrics.functional import pairwise_cosine_similarity - -from torch_geometric.data import Data, HeteroData -from torch_geometric.distributed import LocalFeatureStore -from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput -from torch_geometric.typing import InputEdges, InputNodes - - -# NOTE: Only compatible with Homogeneous graphs for now -class KNNRAGFeatureStore(LocalFeatureStore): - def __init__(self, enc_model: Type[Module], - model_kwargs: Optional[Dict[str, - Any]] = None, *args, **kwargs): - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") - self.enc_model = enc_model(*args, **kwargs).to(self.device) - self.enc_model.eval() - self.model_kwargs = \ - model_kwargs if model_kwargs is not None else dict() - super().__init__() - - @property - def x(self) -> Tensor: - return self.get_tensor(group_name=None, attr_name='x') - - @property - def edge_attr(self) -> Tensor: - return self.get_tensor(group_name=(None, None), attr_name='edge_attr') - - def retrieve_seed_nodes(self, query: Any, k_nodes: int = 5) -> InputNodes: - return next(self._retrieve_seed_nodes_batch([query], k_nodes)) - - def _retrieve_seed_nodes_batch(self, query: Iterable[Any], - k_nodes: int) -> Iterator[InputNodes]: - if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): - raise NotImplementedError - - query_enc = self.enc_model.encode(query, - **self.model_kwargs).to(self.device) - prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) - topk = min(k_nodes, len(self.x)) - for q in prizes: - _, indices = torch.topk(q, topk, largest=True) - yield indices - - def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: - return next(self._retrieve_seed_edges_batch([query], k_edges)) - - def _retrieve_seed_edges_batch(self, query: Iterable[Any], - k_edges: int) -> Iterator[InputEdges]: - if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): - raise NotImplementedError - - query_enc = self.enc_model.encode(query, - **self.model_kwargs).to(self.device) - - prizes = pairwise_cosine_similarity(query_enc, - self.edge_attr.to(self.device)) - topk = min(k_edges, len(self.edge_attr)) - for q in prizes: - _, indices = torch.topk(q, topk, largest=True) - yield indices - - def load_subgraph( - self, sample: Union[SamplerOutput, HeteroSamplerOutput] - ) -> Union[Data, HeteroData]: - - if isinstance(sample, HeteroSamplerOutput): - raise NotImplementedError - - # NOTE: torch_geometric.loader.utils.filter_custom_store can be used - # here if it supported edge features - node_id = sample.node - edge_id = sample.edge - edge_index = torch.stack((sample.row, sample.col), dim=0) - x = self.x[node_id] - edge_attr = self.edge_attr[edge_id] - - return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - node_idx=node_id, edge_idx=edge_id) - - -class SentenceTransformerFeatureStore(KNNRAGFeatureStore): - def __init__(self, *args, **kwargs): - kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') - super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py deleted file mode 100644 index d4bb628dec56..000000000000 --- a/examples/llm_plus_gnn/rag_generate.py +++ /dev/null @@ -1,75 +0,0 @@ -# %% -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import RAGQueryLoader -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd - -# %% -ds = UpdatedWebQSPDataset("small_dataset", force_reload=True, limit=20) - -# %% -triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) - -# %% -questions = ds.raw_dataset['question'] -questions - -# %% -ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) -num_edges = len(ds.indexer._edges) - -# %% -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) - -# %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() - -# %% -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}) - -# %% -# Accuracy Metrics to be added to Profiler -def _eidx_helper(subg: Data, ground_truth: Data): - subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx - if isinstance(subg_eidx, torch.Tensor): - subg_eidx = subg_eidx.tolist() - if isinstance(gt_eidx, torch.Tensor): - gt_eidx = gt_eidx.tolist() - subg_e = set(subg_eidx) - gt_e = set(gt_eidx) - return subg_e, gt_e -def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - total_e = set(range(num_edges)) - tp = len(subg_e & gt_e) - tn = len(total_e-(subg_e | gt_e)) - return (tp+tn)/num_edges -def check_retrieval_precision(subg: Data, ground_truth: Data): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - return len(subg_e & gt_e) / len(subg_e) -def check_retrieval_recall(subg: Data, ground_truth: Data): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - return len(subg_e & gt_e) / len(gt_e) - -# %% -retrieval_stats = {"precision": [], "recall": [], "accuracy": []} -subgs = [] -for subg, gt in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): - retrieval_stats["precision"].append(check_retrieval_precision(subg, gt)) - retrieval_stats["recall"].append(check_retrieval_recall(subg, gt)) - retrieval_stats["accuracy"].append(check_retrieval_accuracy(subg, gt, num_edges)) - subgs.append(subg) - -pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') -torch.save(subgs, 'subg_results.pt') - diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/rag_generate_multihop.py deleted file mode 100644 index 3640dca39166..000000000000 --- a/examples/llm_plus_gnn/rag_generate_multihop.py +++ /dev/null @@ -1,38 +0,0 @@ -# %% -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import RAGQueryLoader -from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd - - -# %% -triplets = torch.load('wikimultihopqa_full_graph.pt') - -# %% -df = pd.read_csv('wikimultihopqa_cleaned.csv') -questions = df['question_text'][:10] - -# %% -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) - -# %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() - -# %% -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=retrieval_via_pcst) - -# %% -subgs = [] -for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): - subgs.append(subg) - -torch.save(subgs, 'subg_results.pt') diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py deleted file mode 100644 index ee780aff0860..000000000000 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import Optional, Union - -import torch -from torch import Tensor - -from torch_geometric.data import FeatureStore -from torch_geometric.distributed import LocalGraphStore -from torch_geometric.sampler import ( - HeteroSamplerOutput, - NeighborSampler, - NodeSamplerInput, - SamplerOutput, -) -from torch_geometric.sampler.neighbor_sampler import NumNeighborsType -from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes - - -class NeighborSamplingRAGGraphStore(LocalGraphStore): - def __init__(self, feature_store: Optional[FeatureStore] = None, - num_neighbors: NumNeighborsType = [1], **kwargs): - self.feature_store = feature_store - self._num_neighbors = num_neighbors - self.sample_kwargs = kwargs - self._sampler_is_initialized = False - super().__init__() - - def _init_sampler(self): - if self.feature_store is None: - raise AttributeError("Feature store not registered yet.") - self.sampler = NeighborSampler(data=(self.feature_store, self), - num_neighbors=self._num_neighbors, - **self.sample_kwargs) - self._sampler_is_initialized = True - - def register_feature_store(self, feature_store: FeatureStore): - self.feature_store = feature_store - self._sampler_is_initialized = False - - def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: - ret = super().put_edge_id(edge_id.contiguous(), *args, **kwargs) - self._sampler_is_initialized = False - return ret - - @property - def edge_index(self): - return self.get_edge_index(*self.edge_idx_args, **self.edge_idx_kwargs) - - def put_edge_index(self, edge_index: EdgeTensorType, *args, - **kwargs) -> bool: - ret = super().put_edge_index(edge_index, *args, **kwargs) - # HACK - self.edge_idx_args = args - self.edge_idx_kwargs = kwargs - self._sampler_is_initialized = False - return ret - - @property - def num_neighbors(self): - return self._num_neighbors - - @num_neighbors.setter - def num_neighbors(self, num_neighbors: NumNeighborsType): - self._num_neighbors = num_neighbors - if hasattr(self, 'sampler'): - self.sampler.num_neighbors = num_neighbors - - def sample_subgraph( - self, seed_nodes: InputNodes, seed_edges: InputEdges, - num_neighbors: Optional[NumNeighborsType] = None - ) -> Union[SamplerOutput, HeteroSamplerOutput]: - if not self._sampler_is_initialized: - self._init_sampler() - if num_neighbors is not None: - self.num_neighbors = num_neighbors - - # FIXME: Right now, only input nodes/edges as tensors are be supported - if not isinstance(seed_nodes, Tensor): - raise NotImplementedError - if not isinstance(seed_edges, Tensor): - raise NotImplementedError - device = seed_nodes.device - - # TODO: Call sample_from_edges for seed_edges - seed_edges = self.edge_index.to(device).T[seed_edges.to( - device)].reshape((-1)) - seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) - - seed_nodes = seed_nodes.unique().contiguous() - node_sample_input = NodeSamplerInput(input_id=None, node=seed_nodes) - out = self.sampler.sample_from_nodes(node_sample_input) - - return out diff --git a/examples/llm_plus_gnn/raw_qsp_dataset.py b/examples/llm_plus_gnn/raw_qsp_dataset.py deleted file mode 100644 index cd6a16e8aed2..000000000000 --- a/examples/llm_plus_gnn/raw_qsp_dataset.py +++ /dev/null @@ -1,139 +0,0 @@ -from typing import Dict, List, Optional - -import torch -from tqdm import tqdm - -from torch_geometric.data import Data, InMemoryDataset -from torch_geometric.datasets.web_qsp_dataset import ( - DataFrame, - WebQSPDataset, - datasets, - retrieval_via_pcst, -) -from torch_geometric.nn.nlp import SentenceTransformer - - -class RawWebQSPDataset(WebQSPDataset): - def __init__( - self, - root: str = "", - force_reload: bool = False, - with_process: bool = False, - with_pcst: bool = False, - limit: Optional[int] = None, - ) -> None: - self.with_process = with_process - self.with_pcst = with_pcst - self.limit = limit - if self.with_process: - super().__init__(root, force_reload) - else: - self._check_dependencies() - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") - super(InMemoryDataset, self).__init__(root, None, None, - force_reload=force_reload) - self._load_raw_data() - - @property - def raw_file_names(self) -> List[str]: - if not self.with_process: - return ["raw_data", "split_idxs"] - else: - return [] - - @property - def processed_file_names(self) -> List[str]: - if self.with_process: - return super().processed_file_names + ["raw_graphs.pt"] - else: - return super().processed_file_names - - def _save_raw_data(self) -> None: - self.raw_dataset.save_to_disk(self.raw_paths[0]) - torch.save(self.split_idxs, self.raw_paths[1]) - - def _load_raw_data(self) -> None: - self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) - self.split_idxs = torch.load(self.raw_paths[1]) - - def download(self) -> None: - super().download() - if not self.with_process: - self._save_raw_data() - - def process(self) -> None: - if self.with_process: - self.model = SentenceTransformer().to(self.device) - self.model.eval() - list_of_graphs = [] - self.raw_graphs = [] - if self.with_pcst: - self.questions = [i["question"] for i in self.raw_dataset] - print("Encoding questions...") - q_embs = self.model.encode(self.questions, batch_size=256) - - print("Encoding graphs...") - limit = self.limit if self.limit else len(self.raw_dataset) - for index in tqdm(range(limit)): - data_i = self.raw_dataset[index] - raw_nodes: Dict[str, int] = {} - raw_edges = [] - for tri in data_i["graph"]: - h, r, t = tri - h = h.lower() - t = t.lower() - if h not in raw_nodes: - raw_nodes[h] = len(raw_nodes) - if t not in raw_nodes: - raw_nodes[t] = len(raw_nodes) - raw_edges.append({ - "src": raw_nodes[h], - "edge_attr": r, - "dst": raw_nodes[t] - }) - nodes = DataFrame( - [{ - "node_id": v, - "node_attr": k - } for k, v in raw_nodes.items()], - columns=["node_id", "node_attr"], - ) - edges = DataFrame(raw_edges, - columns=["src", "edge_attr", "dst"]) - # encode nodes - nodes.node_attr = nodes.node_attr.fillna("") - x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) - # encode edges - edge_attr = self.model.encode(edges.edge_attr.tolist(), - batch_size=256) - edge_index = torch.LongTensor( - [edges.src.tolist(), - edges.dst.tolist()]) - question = f"Question: {data_i['question']}\nAnswer: " - label = ("|").join(data_i["answer"]).lower() - raw_graph = Data( - x=x, - edge_index=edge_index, - edge_attr=edge_attr, - num_nodes=len(nodes), - ).to("cpu") - self.raw_graphs.append(raw_graph) - if self.with_pcst: - psct_subgraph, desc = retrieval_via_pcst( - raw_graph, - q_embs[index], - nodes, - edges, - topk=3, - topk_e=5, - cost_e=0.5, - ) - psct_subgraph["question"] = question - psct_subgraph["label"] = label - psct_subgraph["desc"] = desc - list_of_graphs.append(psct_subgraph.to("cpu")) - else: - list_of_graphs.append(raw_graph) - torch.save(self.raw_graphs, self.processed_paths[-1]) - self.save(list_of_graphs, self.processed_paths[0]) diff --git a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb b/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb deleted file mode 100644 index 0b6ed8b5635e..000000000000 --- a/examples/llm_plus_gnn/test_eval_large_graph_indexer.ipynb +++ /dev/null @@ -1,737 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from typing import List\n", - "from torch_geometric.data import InMemoryDataset\n", - "from torch_geometric.distributed.local_graph_store import LocalGraphStore\n", - "from torch_geometric.datasets.web_qsp_dataset import *\n", - "import torch\n", - "import datasets\n", - "import time\n", - "from unittest.mock import patch" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from raw_qsp_dataset import RawWebQSPDataset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n", - "Done!\n" - ] - } - ], - "source": [ - "dataset = RawWebQSPDataset(force_reload=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Dataset({\n", - " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", - " num_rows: 4700\n", - "})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.raw_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "from large_graph_indexer import LargeGraphIndexer, TripletLike" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def preprocess_triplet(triplet: TripletLike):\n", - " h, r, t = triplet\n", - " return h.lower(), r, t.lower()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test basic collation" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "indexer = LargeGraphIndexer.from_triplets(dataset.raw_dataset[0]['graph'], pre_transform=preprocess_triplet)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "indexer2 = LargeGraphIndexer.from_triplets(dataset.raw_dataset[1]['graph'], pre_transform=preprocess_triplet)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "bigger_indexer = LargeGraphIndexer.collate([indexer, indexer2])" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(indexer._nodes) + len(indexer2._nodes) - len(indexer._nodes.keys() & indexer2._nodes.keys()) == len(bigger_indexer._nodes)\n", - "assert len(indexer._edges) + len(indexer2._edges) - len(indexer._edges.keys() & indexer2._edges.keys()) == len(bigger_indexer._edges)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(set(bigger_indexer._nodes.values())) == len(bigger_indexer._nodes)\n", - "assert len(set(bigger_indexer._edges.values())) == len(bigger_indexer._edges)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "for node in indexer._nodes.keys():\n", - " assert indexer.node_attr['pid'][indexer._nodes[node]] == node, f'{node} is not {indexer.node_attr[\"pid\"][indexer._nodes[node]]}'" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm\n", - "from multiprocessing import Pool" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test collation on entire dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "LIMIT=100\n", - "def get_next_graph(dataset, limit=None):\n", - " i = 0\n", - " for ds in dataset:\n", - " if i == limit:\n", - " break\n", - " yield ds['graph']\n", - " i += 1\n", - "graphs = get_next_graph(dataset.raw_dataset, limit=LIMIT)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [00:01<00:00, 55.69it/s]\n" - ] - } - ], - "source": [ - "indexers = []\n", - "def from_trips_with_pretransform(triplets):\n", - " return LargeGraphIndexer.from_triplets(triplets, pre_transform=preprocess_triplet)\n", - "with Pool(40) as p:\n", - " indexers = list(tqdm.tqdm(p.imap(from_trips_with_pretransform, graphs), total=len(dataset.raw_dataset) if LIMIT is None else LIMIT))" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.6796014308929443" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "start = time.time()\n", - "big_indexer = LargeGraphIndexer.collate(indexers)\n", - "time.time()-start" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Naive method, concatenating all the triplets" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "from itertools import chain" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "large_graph_dataset = chain.from_iterable(get_next_graph(dataset.raw_dataset, limit=LIMIT))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "# This is just for TQDM to work well\n", - "total_size = 0\n", - "#for g in tqdm.tqdm(get_next_graph(dataset.raw_dataset), total=4700):\n", - "# total_size += len(g)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "423763it [00:02, 206877.03it/s]\n" - ] - } - ], - "source": [ - "large_indexer = LargeGraphIndexer.from_triplets(tqdm.tqdm(large_graph_dataset, total=total_size), pre_transform=preprocess_triplet)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "assert set(large_indexer._nodes) == set(big_indexer._nodes)\n", - "assert set(large_indexer._edges) == set(big_indexer._edges)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Phase I: Indexing Large Graph" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "node_attributes = list(big_indexer.get_unique_node_features())" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "edge_attributes = list(big_indexer.get_unique_edge_features(\"r\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "105413" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(node_attributes)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3095" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(edge_attributes)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "from torch_geometric.nn.nlp import SentenceTransformer" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "SentenceTransformer(model_name=sentence-transformers/all-roberta-large-v1)" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer().to(device)\n", - "model.eval()" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cuda\n" - ] - } - ], - "source": [ - "print(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "from more_itertools import chunked" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "nodes_to_process, edges_to_process = len(node_attributes), len(edge_attributes)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 32" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/3294 [00:00 4\u001b[0m node_embs\u001b[38;5;241m.\u001b[39mappend(\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m(nbatch, batch_size\u001b[38;5;241m=\u001b[39mBATCH_SIZE)\u001b[38;5;241m.\u001b[39mcpu())\n\u001b[1;32m 5\u001b[0m node_embs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat(node_embs, \u001b[38;5;241m0\u001b[39m)\n", - "File \u001b[0;32m~/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/torch/nn/modules/module.py:1709\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1708\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1709\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mAttributeError\u001b[0m: 'SentenceTransformer' object has no attribute 'encode'" - ] - } - ], - "source": [ - "# Indexing graph features\n", - "node_embs = []\n", - "for nbatch in tqdm.tqdm(chunked(node_attributes, BATCH_SIZE), total=nodes_to_process//BATCH_SIZE):\n", - " node_embs.append(model.encode(nbatch, batch_size=BATCH_SIZE).cpu())\n", - "node_embs = torch.cat(node_embs, 0)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([105413, 1024])" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "node_embs.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "big_indexer.add_node_feature(new_feature_name=\"x\", new_feature_vals=node_embs)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Iterable, Callable, Optional\n", - "from large_graph_indexer import TripletLike, ordered_set\n", - "from torch_geometric.typing import FeatureTensorType\n", - "from torch_geometric.data import Data" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "97it [00:01, 50.34it/s] \n" - ] - } - ], - "source": [ - "# Indexing graph features\n", - "edge_embs = []\n", - "for ebatch in tqdm.tqdm(chunked(edge_attributes, BATCH_SIZE), total=edges_to_process//BATCH_SIZE):\n", - " edge_embs.append(model.encode(ebatch, batch_size=BATCH_SIZE).cpu())\n", - "edge_embs = torch.cat(edge_embs, 0)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([3095, 1024])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "edge_embs.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "big_indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_embs, map_from_feature=\"r\")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "from large_graph_indexer import get_features_for_triplets" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "import networkx as nx" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "first_10_trips = dataset.raw_dataset[:10]['graph']" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 10/10 [00:01<00:00, 6.60it/s]\n" - ] - } - ], - "source": [ - "#TODO: Parallelize\n", - "first_10 = [get_features_for_triplets(big_indexer, trip_lst, pre_transform=preprocess_triplet) for trip_lst in tqdm.tqdm(first_10_trips)]" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding graphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 10/10 [00:00<00:00, 56.01it/s]\n", - "Done!\n" - ] - } - ], - "source": [ - "# Grab the first few samples from the old ds to test with LargeGraphIndexer\n", - "raw_mock = None\n", - "with patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as mock_method:\n", - " mock_method.return_value: torch.Tensor = torch.zeros(256, 1024)\n", - "old_dataset = RawWebQSPDataset(force_reload=True, with_process=True, limit=10)\n", - " raw_mock = mock_method" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", - " def _sorted_tensors_are_close(tensor1, tensor2):\n", - " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", - " def _graphs_are_same(tensor1, tensor2):\n", - " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", - " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", - " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", - " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ds in zip(old_dataset, first_10):\n", - " print(results_are_close_enough(*ds))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test Saving and Loading and eq" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "big_indexer.save('indexer')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert big_indexer == LargeGraphIndexer.from_disk('indexer')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb b/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb deleted file mode 100644 index 59480bbc5463..000000000000 --- a/examples/llm_plus_gnn/test_eval_updated_webqsp.ipynb +++ /dev/null @@ -1,559 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from updated_qsp_dataset import UpdatedWebQSPDataset\n", - "from raw_qsp_dataset import RawWebQSPDataset\n", - "from unittest.mock import patch\n", - "from torch_geometric.data import Data\n", - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def tokenizer_call_iter():\n", - " a, batch_size = [], 256\n", - " i = 0\n", - " while True:\n", - " rv = torch.Tensor([list(range(i, i+len(a)))]).T\n", - " i += len(a)\n", - " a = yield rv\n", - " if a is None:\n", - " a = []\n", - "gen = tokenizer_call_iter()\n", - "next(gen)\n", - "def tokenizer_call(a, batch_size):\n", - " return gen.send(a)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding questions...\n", - "Encoding graphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 46.95it/s]\n", - "Done!\n" - ] - } - ], - "source": [ - "old_token_mock, old_ret_mock = None, None\n", - "with (patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock,\n", - " patch(\"raw_qsp_dataset.retrieval_via_pcst\") as pcst_mock):\n", - " model_mock.side_effect =tokenizer_call\n", - " pcst_mock.return_value = Data(), \"\"\n", - " old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", - " old_token_mock = model_mock\n", - " old_ret_mock = pcst_mock" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "# Create reference list for all tokenizer call unique values\n", - "TOKEN_CALLS = set()\n", - "for call in model_mock.call_args_list:\n", - " text_lst = call[0][0]\n", - " TOKEN_CALLS |= set(text_lst)\n", - "TOKEN_CALLS_MAP = list(TOKEN_CALLS)\n", - "TOKEN_CALLS_REVERSE_MAP = {v: i for i, v in enumerate(TOKEN_CALLS_MAP)}" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def tokenizer_from_map_iter():\n", - " a = []\n", - " while True:\n", - " rv = torch.Tensor([TOKEN_CALLS_REVERSE_MAP[t] for t in a])\n", - " a = yield rv\n", - " if a is None:\n", - " a = []\n", - "gen = tokenizer_from_map_iter()\n", - "next(gen)\n", - "def tokenizer_from_map(a, batch_size):\n", - " return gen.send(a)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding questions...\n", - "Encoding graphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 45.33it/s]\n", - "Done!\n" - ] - } - ], - "source": [ - "old_token_mock, old_ret_mock = None, None\n", - "with (patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock,\n", - " patch(\"raw_qsp_dataset.retrieval_via_pcst\") as pcst_mock):\n", - " model_mock.side_effect =tokenizer_from_map\n", - " pcst_mock.return_value = Data(), \"\"\n", - " old_dataset = RawWebQSPDataset(root='old_dataset', force_reload=True, limit=2, with_process=True, with_pcst=True)\n", - " old_token_mock = model_mock\n", - " old_ret_mock = pcst_mock" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding graph...\n", - "Encoding questions...\n", - "Retrieving subgraphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 6.98it/s]\n", - "Done!\n" - ] - } - ], - "source": [ - "new_token_mock, new_ret_mock = None, None\n", - "with (patch(\"torch_geometric.datasets.web_qsp_dataset.SentenceTransformer.encode\") as model_mock,\n", - " patch(\"updated_qsp_dataset.retrieval_via_pcst\") as pcst_mock):\n", - " model_mock.side_effect = tokenizer_from_map\n", - " pcst_mock.return_value = Data(), \"\"\n", - " old_dataset = UpdatedWebQSPDataset(root='old_dataset', force_reload=True, limit=2, whole_graph_retrieval=False)\n", - " new_token_mock = model_mock\n", - " new_ret_mock = pcst_mock" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [], - "source": [ - "old_nodes_called = [old_ret_mock.call_args_list[i][0][0].x.int().numpy() for i in range(len(old_ret_mock.call_args_list))]\n", - "old_edges_called = [old_ret_mock.call_args_list[i][0][0].edge_attr.int().numpy() for i in range(len(old_ret_mock.call_args_list))]\n", - "old_question_called = [old_ret_mock.call_args_list[i][0][1].int().numpy() for i in range(len(old_ret_mock.call_args_list))]" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "new_nodes_called = [new_ret_mock.call_args_list[i][0][0].x.int().numpy() for i in range(len(new_ret_mock.call_args_list))]\n", - "new_edges_called = [new_ret_mock.call_args_list[i][0][0].edge_attr.int().numpy() for i in range(len(new_ret_mock.call_args_list))]\n", - "new_question_called = [new_ret_mock.call_args_list[i][0][1].int().numpy() for i in range(len(new_ret_mock.call_args_list))]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "True\n", - "True\n", - "True\n", - "True\n", - "True\n", - "True\n" - ] - } - ], - "source": [ - "for i in range(len(old_ret_mock.call_args_list)):\n", - " print(set(old_nodes_called[i]) == set(new_nodes_called[i]))\n", - " print(set(old_edges_called[i]) == set(new_edges_called[i]))\n", - " print(old_question_called[i] == new_question_called[i])" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[array(3073, dtype=int32), array(2115, dtype=int32)]" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "old_question_called" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([ 0., 3971., 3972., ..., 8044., 8046., 5478.]),\n", - " tensor([ 12., 3985., 3987., ..., 3953., 8035., 8038.])]" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_nodes_called" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "new_dataset = UpdatedWebQSPDataset(root='updated_dataset',force_reload=True, limit=2, whole_graph_retrieval=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Data(x=[12, 1024], edge_index=[2, 12], edge_attr=[12, 1024], question='Question: what is the name of justin bieber brother\n", - "Answer: ', label='jaxon bieber', desc='node_id,node_attr\n", - "15,justin bieber\n", - "151,pattie mallette\n", - "286,english language\n", - "294,jaxon bieber\n", - "346,yves bole\n", - "356,jeremy bieber\n", - "452,jazmyn bieber\n", - "545,m.0wfn4pm\n", - "551,m.0gxnnwp\n", - "933,m.0gxnnwc\n", - "1032,this is justin bieber\n", - "1359,m.0129jzth\n", - "\n", - "src,edge_attr,dst\n", - "346,people.person.languages,286\n", - "1032,film.film.language,286\n", - "346,influence.influence_node.influenced_by,15\n", - "151,people.person.children,15\n", - "294,people.person.parents,356\n", - "545,people.sibling_relationship.sibling,151\n", - "933,people.sibling_relationship.sibling,452\n", - "1359,people.sibling_relationship.sibling,346\n", - "551,people.sibling_relationship.sibling,294\n", - "15,people.person.sibling_s,933\n", - "933,people.sibling_relationship.sibling,15\n", - "551,people.sibling_relationship.sibling,15\n", - "', num_nodes=12)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "old_dataset[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Data(x=[15, 1024], edge_index=[2, 20], edge_attr=[20, 1024], question='Question: what is the name of justin bieber brother\n", - "Answer: ', label='jaxon bieber', desc='index,node_id,node_attr\n", - "54,54,m.0gxnp5m\n", - "307,307,m.0129jzth\n", - "391,391,pattie mallette\n", - "650,650,english language\n", - "837,837,justin bieber\n", - "934,934,this is justin bieber\n", - "1022,1022,m.0gxnp5x\n", - "1123,1123,jeremy bieber\n", - "1149,1149,m.0gxnnwp\n", - "1847,1847,m.0gxnnwc\n", - "2294,2294,yves bole\n", - "2702,2702,m.0gxnp5d\n", - "2719,2719,jazmyn bieber\n", - "2880,2880,m.0wfn4pm\n", - "2882,2882,jaxon bieber\n", - "\n", - "src,edge_attr,dst\n", - "2294,people.person.languages,650\n", - "934,film.film.language,650\n", - "2294,influence.influence_node.influenced_by,837\n", - "391,people.person.children,837\n", - "2882,people.person.parents,1123\n", - "2880,people.sibling_relationship.sibling,391\n", - "2882,people.person.sibling_s,1149\n", - "837,base.popstra.celebrity.hangout,1022\n", - "1847,people.sibling_relationship.sibling,2719\n", - "307,people.sibling_relationship.sibling,2294\n", - "391,people.person.sibling_s,2880\n", - "1149,people.sibling_relationship.sibling,2882\n", - "2294,people.person.sibling_s,307\n", - "837,people.person.sibling_s,1847\n", - "837,base.popstra.celebrity.hangout,2702\n", - "2719,people.person.sibling_s,1847\n", - "1847,people.sibling_relationship.sibling,837\n", - "837,people.person.sibling_s,1149\n", - "837,base.popstra.celebrity.hangout,54\n", - "1149,people.sibling_relationship.sibling,837\n", - "', num_nodes=15)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_dataset[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Data(x=[1709, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], num_nodes=1709)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "old_dataset.raw_graphs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Data(x=[1709, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], num_nodes=1709, pid=[1709], e_pid=[9088], node_idx=[1709], edge_idx=[9088])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_dataset.raw_graphs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "import networkx as nx\n", - "from torch_geometric.data import Data\n", - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.9):\n", - " def _sorted_tensors_are_close(tensor1, tensor2):\n", - " vals = torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1)\n", - " return torch.all(vals > thresh)\n", - " def _graphs_are_same(tensor1, tensor2):\n", - " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", - " val = _sorted_tensors_are_close(ground_truth.x, new_method.x)\n", - " val &= _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr)\n", - " val &= _graphs_are_same(ground_truth.edge_index, new_method.edge_index)\n", - " return val" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(False)" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "results_are_close_enough(old_dataset.raw_graphs[0], new_dataset.raw_graphs[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "old_dataset.questions[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "new_dataset.questions[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "torch.sort(new_dataset.raw_graphs[0].edge_attr, 0)[0].unique(dim=0).size(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "torch.sort(old_dataset.raw_graphs[0].edge_attr, 0)[0].unique(dim=0).size()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/test_g_retriever.ipynb b/examples/llm_plus_gnn/test_g_retriever.ipynb deleted file mode 100644 index fb507b0332ed..000000000000 --- a/examples/llm_plus_gnn/test_g_retriever.ipynb +++ /dev/null @@ -1,363 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.10/dist-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from g_retriever import benchmark_models, get_loss, inference_step\n", - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from torch_geometric.nn.models import GRetriever" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "hidden_channels = 1024\n", - "num_gnn_layers = 1\n", - "lr=1e-5\n", - "epochs=2\n", - "batch_size=8\n", - "eval_batch_size=16" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "ds = UpdatedWebQSPDataset('test_script', limit=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "model_names = [\"GAT_1\", \"MLP_1\"]\n", - "model_classes = [GRetriever, GRetriever]\n", - "model_kwargs = [dict(llm_to_use=\"TinyLlama/TinyLlama-1.1B-Chat-v0.1\", num_llm_params=1, gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers, mlp_out_dim=2048)] * 2" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.10/dist-packages/torch_geometric/nn/nlp/llm.py:102: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n", - " self.autocast_context = torch.cuda.amp.autocast(dtype=dtype)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Model GAT_1 appears to already exist.\n", - "Would you like to retrain?\n", - "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n", - "16 8 8\n", - "Total Prep Time (prep_time) = 1.87\n", - "Training beginning...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch: 1|2: 100%|██████████| 2/2 [00:00<00:00, 2.27it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1|2,Train Loss (Epoch Mean): 3.2131307125091553\n", - "Epoch: 1|2, Val Loss: 2.994086265563965\n", - "Checkpointing best val loss model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch: 2|2: 100%|██████████| 2/2 [00:00<00:00, 2.58it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 2|2,Train Loss (Epoch Mean): 2.8135846853256226\n", - "Epoch: 2|2, Val Loss: 2.8492681980133057\n", - "Checkpointing best val loss model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.10/dist-packages/torch/cuda/memory.py:350: FutureWarning: torch.cuda.reset_max_memory_allocated now calls torch.cuda.reset_peak_memory_stats, which resets /all/ peak memory stats.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Final Evaluation...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1/1 [00:00<00:00, 1.47it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Label: dilma rousseff\n", - "Pred: ['[/SRC] [/EDGE] [/SRC] [/EDGE] [/SRC] [/EDGE] [/']\n", - "Exception: unterminated character set at position 45\n", - "------------------\n", - "Label: paris\n", - "Pred: ['[/QU] [/QC] [/Q] [/QA] [/QB] [/QC] [/QD']\n", - "Exception: unterminated character set at position 35\n", - "------------------\n", - "Label: ice hockey\n", - "Pred: ['[/SRC] [/EDGE] [/SRC,EDGE] [/EDGE,SRC] [/EDGE,ED']\n", - "Exception: unterminated character set at position 39\n", - "------------------\n", - "Label: chaz bono|elijah blue allman\n", - "Pred: ['A: You can use the following code:\\nimport pprint\\n\\ndef load_data():\\n with open(\"data.txt']\n", - "Exception: missing ), unterminated subpattern at position 80 (line 5, column 14)\n", - "------------------\n", - "Hit: 0.0000\n", - "Precision: 0.0000\n", - "Recall: 0.0000\n", - "F1: 0.0000\n", - "Test Acc 0.0\n", - "Saving Model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1/1 [00:04<00:00, 4.01s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving eval output for downstream demo...\n", - "Done!\n", - "E2E time (e2e_time) = 14.79 seconds\n", - "E2E tme minus Prep Time = 12.919999999999998 seconds\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved outputs for GAT_1 have been found.\n", - "Would you like to redo?\n", - "Model MLP_1 appears to already exist.\n", - "Would you like to retrain?\n", - "Setting up 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' with configuration: {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n", - "16 8 8\n", - "Total Prep Time (prep_time) = 1.76\n", - "Training beginning...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch: 1|2: 100%|██████████| 2/2 [00:00<00:00, 2.79it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1|2,Train Loss (Epoch Mean): 3.2048009634017944\n", - "Epoch: 1|2, Val Loss: 3.015418291091919\n", - "Checkpointing best val loss model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch: 2|2: 100%|██████████| 2/2 [00:00<00:00, 2.60it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 2|2,Train Loss (Epoch Mean): 2.8109872341156006\n", - "Epoch: 2|2, Val Loss: 2.8776462078094482\n", - "Checkpointing best val loss model...\n", - "Final Evaluation...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1/1 [00:00<00:00, 1.55it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Label: dilma rousseff\n", - "Pred: ['[/SRC] [/EDGE] [/SRC] [/EDGE] [/SRC] [/EDGE] [/']\n", - "Exception: unterminated character set at position 45\n", - "------------------\n", - "Label: paris\n", - "Pred: ['[/QUERY] [/S] [/C] [/A] [/P] [/F] [/R] [/L']\n", - "Exception: unterminated character set at position 39\n", - "------------------\n", - "Label: ice hockey\n", - "Pred: ['[/SRC] [/EDR] [/Q] [/P] [/T] [/E] [/L] [/']\n", - "Exception: unterminated character set at position 39\n", - "------------------\n", - "Hit: 0.0000\n", - "Precision: 0.0000\n", - "Recall: 0.0000\n", - "F1: 0.0000\n", - "Test Acc 0.0\n", - "Saving Model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1/1 [00:04<00:00, 4.05s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving eval output for downstream demo...\n", - "Done!\n", - "E2E time (e2e_time) = 15.26 seconds\n", - "E2E tme minus Prep Time = 13.5 seconds\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved outputs for MLP_1 have been found.\n", - "Would you like to redo?\n", - "Hallucinates for pure_llm:\n", - "pure_llm\n", - "True 5\n", - "skip 2\n", - "False 1\n", - "Name: count, dtype: int64\n", - "Hallucinates for tuned_llm:\n", - "tuned_llm\n", - "True 6\n", - "skip 1\n", - "False 1\n", - "Name: count, dtype: int64\n", - "Hallucinates for GAT_1:\n", - "GAT_1\n", - "skip 4\n", - "True 2\n", - "False 2\n", - "Name: count, dtype: int64\n", - "Hallucinates for MLP_1:\n", - "MLP_1\n", - "skip 3\n", - "True 3\n", - "False 2\n", - "Name: count, dtype: int64\n" - ] - } - ], - "source": [ - "benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/test_nvtx_rag_backend.py deleted file mode 100644 index b0f89bfcca61..000000000000 --- a/examples/llm_plus_gnn/test_nvtx_rag_backend.py +++ /dev/null @@ -1,101 +0,0 @@ -# %% -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import rag_loader -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -from torch_geometric.profile.nvtx import nvtxit -import torch -import argparse -from typing import Tuple - -# %% -# Patch FeatureStore and GraphStore - -SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_nodes) -SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_edges) -SentenceTransformerFeatureStore.load_subgraph = nvtxit()(SentenceTransformerFeatureStore.load_subgraph) -NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()(NeighborSamplingRAGGraphStore.sample_subgraph) -rag_loader.RAGQueryLoader.query = nvtxit()(rag_loader.RAGQueryLoader.query) - -# %% -ds = UpdatedWebQSPDataset("small_ds_1", force_reload=True, limit=10) - -# %% -triplets = list(chain.from_iterable((d['graph'] for d in ds.raw_dataset))) - -# %% -questions = ds.raw_dataset['question'] - -# %% -ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) -num_edges = len(ds.indexer._edges) - -# %% -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer().to(device) - -# %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() - -# %% -from torch_geometric.datasets.updated_web_qsp_dataset import retrieval_via_pcst - -@nvtxit() -def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: - q_emb = model.encode(query) - textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() - textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() - out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) - out_graph["desc"] = desc - return graph - -# %% -query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}, local_filter=apply_retrieval_via_pcst) - -# %% -# Accuracy Metrics to be added to Profiler -def _eidx_helper(subg: Data, ground_truth: Data): - subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx - if isinstance(subg_eidx, torch.Tensor): - subg_eidx = subg_eidx.tolist() - if isinstance(gt_eidx, torch.Tensor): - gt_eidx = gt_eidx.tolist() - subg_e = set(subg_eidx) - gt_e = set(gt_eidx) - return subg_e, gt_e -def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - total_e = set(range(num_edges)) - tp = len(subg_e & gt_e) - tn = len(total_e-(subg_e | gt_e)) - return (tp+tn)/num_edges -def check_retrieval_precision(subg: Data, ground_truth: Data): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - return len(subg_e & gt_e) / len(subg_e) -def check_retrieval_recall(subg: Data, ground_truth: Data): - subg_e, gt_e = _eidx_helper(subg, ground_truth) - return len(subg_e & gt_e) / len(gt_e) - - -# %% - -@nvtxit() -def _run_eval(): - for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): - print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--capture-torch-kernels", "-k", action="store_true") - args = parser.parse_args() - if args.capture_torch_kernels: - with torch.autograd.profiler.emit_nvtx(): - _run_eval() - else: - _run_eval() \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py deleted file mode 100644 index 6204fe603b84..000000000000 --- a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py +++ /dev/null @@ -1,27 +0,0 @@ -from torch_geometric.datasets import updated_web_qsp_dataset -from torch_geometric.profile import nvtxit -import torch -import argparse - -# Apply Patches -updated_web_qsp_dataset.UpdatedWebQSPDataset.process = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset.process) -updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph) -updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs) -updated_web_qsp_dataset.SentenceTransformer.encode = nvtxit()(updated_web_qsp_dataset.SentenceTransformer.encode) -updated_web_qsp_dataset.retrieval_via_pcst = nvtxit()(updated_web_qsp_dataset.retrieval_via_pcst) - -updated_web_qsp_dataset.get_features_for_triplets_groups = nvtxit()(updated_web_qsp_dataset.get_features_for_triplets_groups) -updated_web_qsp_dataset.LargeGraphIndexer.get_node_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features) -updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features) -updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter) -updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--capture-torch-kernels", "-k", action="store_true") - args = parser.parse_args() - if args.capture_torch_kernels: - with torch.autograd.profiler.emit_nvtx(): - ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) - else: - ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py deleted file mode 100644 index 1db8e5934728..000000000000 --- a/examples/llm_plus_gnn/test_nvtx_webqsp.py +++ /dev/null @@ -1,19 +0,0 @@ -from torch_geometric.datasets import web_qsp_dataset -from torch_geometric.profile import nvtxit -import torch -import argparse - -# Apply Patches -web_qsp_dataset.retrieval_via_pcst = nvtxit()(web_qsp_dataset.retrieval_via_pcst) -web_qsp_dataset.WebQSPDataset.process = nvtxit()(web_qsp_dataset.WebQSPDataset.process) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--capture-torch-kernels", "-k", action="store_true") - args = parser.parse_args() - if args.capture_torch_kernels: - with torch.autograd.profiler.emit_nvtx(): - ds = web_qsp_dataset.WebQSPDataset('baseline') - else: - ds = web_qsp_dataset.WebQSPDataset('baseline') \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_profiling.ipynb b/examples/llm_plus_gnn/test_profiling.ipynb deleted file mode 100644 index 8bba158b70d2..000000000000 --- a/examples/llm_plus_gnn/test_profiling.ipynb +++ /dev/null @@ -1,2707 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from torch_geometric.nn import GRetriever\n", - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from torch_geometric.profile.profiler import Profiler\n", - "from torch_geometric.profile import profileit, timeit, nvtxit\n", - "from g_retriever import train, get_loss, inference_step\n", - "import time\n", - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "ds = UpdatedWebQSPDataset(root=\"profiled_ds\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading TinyLlama/TinyLlama-1.1B-Chat-v0.1\n", - "Setting up TinyLlama/TinyLlama-1.1B-Chat-v0.1 w/ kwargs = {'revision': 'main', 'max_memory': {0: '22GiB'}, 'low_cpu_mem_usage': True, 'device_map': 'auto', 'torch_dtype': torch.bfloat16}\n" - ] - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = GRetriever(\n", - " llm_to_use=\"TinyLlama/TinyLlama-1.1B-Chat-v0.1\",\n", - " num_llm_params=1, # 1 Billion\n", - " gnn_hidden_channels=1024,\n", - " num_gnn_layers=4,\n", - " mlp_out_dim=2048,\n", - ").to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "@nvtxit(\"test_inference_wrapper\")\n", - "def inference_wrapper(*args, **kwargs):\n", - " return inference_step(*args, **kwargs)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'function' object has no attribute 'named_parameters'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[5], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m----> 2\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43msince\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden_channels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1024\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_gnn_layers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m8\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_batch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1e-5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_loss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minference_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minference_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_wrapper\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/pyg/pytorch_geometric/examples/llm_plus_gnn/g_retriever.py:177\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model, dataset, checkpointing, tiny_llama)\u001b[0m\n\u001b[1;32m 174\u001b[0m model_save_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mllm\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 176\u001b[0m \u001b[38;5;66;03m# Step 3 Set Optimizer\u001b[39;00m\n\u001b[0;32m--> 177\u001b[0m params \u001b[38;5;241m=\u001b[39m [p \u001b[38;5;28;01mfor\u001b[39;00m _, p \u001b[38;5;129;01min\u001b[39;00m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnamed_parameters\u001b[49m() \u001b[38;5;28;01mif\u001b[39;00m p\u001b[38;5;241m.\u001b[39mrequires_grad]\n\u001b[1;32m 178\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdamW([\n\u001b[1;32m 179\u001b[0m {\n\u001b[1;32m 180\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparams\u001b[39m\u001b[38;5;124m'\u001b[39m: params,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 183\u001b[0m },\n\u001b[1;32m 184\u001b[0m ], betas\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0.9\u001b[39m, \u001b[38;5;241m0.95\u001b[39m))\n\u001b[1;32m 185\u001b[0m grad_steps \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m\n", - "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'named_parameters'" - ] - } - ], - "source": [ - "start = time.time()\n", - "result = train(since=start, num_epochs=1,dataset=ds, hidden_channels=1024, num_gnn_layers=4, batch_size=8, eval_batch_size=16, lr=1e-5, loss_fn=get_loss, inference_fn=inference_wrapper, model=model)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "@profileit(\"cuda\")\n", - "def profile_wrapper(model: torch.nn.Module, device_tensor: torch.Tensor, **kwargs):\n", - " kwargs['model'] = model\n", - " return train(**kwargs)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total Prep Time (prep_time) = 1.93\n", - "Training beginning...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch: 1|1: 100%|██████████| 353/353 [03:09<00:00, 1.87it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1|1,Train Loss (Epoch Mean): 1.6746924656308744\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/miniconda3/envs/pyg-local-dev/lib/python3.10/site-packages/torch/cuda/memory.py:330: FutureWarning: torch.cuda.reset_max_memory_allocated now calls torch.cuda.reset_peak_memory_stats, which resets /all/ peak memory stats.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1|1, Val Loss: 1.5635125488042831\n", - "Final Evaluation...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 102/102 [02:23<00:00, 1.42s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Label: actor|businessperson|rapper|film director|musician|singer|songwriter|guitarist\n", - "Pred: ['[/S] [/A] [/R] [/P] [/T] [/F] [']\n", - "Exception: unterminated character set at position 36\n", - "------------------\n", - "Hit: 0.3239\n", - "Precision: 0.2031\n", - "Recall: 0.2043\n", - "F1: 0.1680\n", - "Test Acc 0.3239090350338045\n", - "Saving Model...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 102/102 [02:29<00:00, 1.46s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving eval output for downstream demo...\n", - "Done!\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "start = time.time()\n", - "result = profile_wrapper(model, torch.Tensor().to(torch.device('cuda')), since=start, num_epochs=1,dataset=ds, hidden_channels=1024, num_gnn_layers=4, batch_size=8, eval_batch_size=16, lr=1e-5, loss_fn=get_loss, inference_fn=inference_step)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CUDAStats(time=347.8926875, max_allocated_gpu=19610.99, max_reserved_gpu=23148.0, max_active_gpu=19610.99, nvidia_smi_free_cuda=7884.72, nvidia_smi_used_cuda=17497.37)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "result[-1]" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'Profiler' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mProfiler\u001b[49m(model\u001b[38;5;241m=\u001b[39mmodel\u001b[38;5;241m.\u001b[39mgraph_encoder, use_cuda\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, profile_memory\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, max_depth\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m prof:\n\u001b[1;32m 2\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m 3\u001b[0m prep_time, dataset, gnn_llm_eval_outs \u001b[38;5;241m=\u001b[39m train(since\u001b[38;5;241m=\u001b[39mstart, num_epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, hidden_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1024\u001b[39m, num_gnn_layers\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m4\u001b[39m, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m8\u001b[39m, eval_batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m16\u001b[39m, lr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1e-5\u001b[39m, loss_fn\u001b[38;5;241m=\u001b[39mget_loss, inference_fn\u001b[38;5;241m=\u001b[39minference_step, dataset\u001b[38;5;241m=\u001b[39mds, model\u001b[38;5;241m=\u001b[39mmodel)\n", - "\u001b[0;31mNameError\u001b[0m: name 'Profiler' is not defined" - ] - } - ], - "source": [ - "with Profiler(model=model.graph_encoder, use_cuda=True, profile_memory=True, max_depth=1) as prof:\n", - " start = time.time()\n", - " prep_time, dataset, gnn_llm_eval_outs = train(since=start, num_epochs=1, hidden_channels=1024, num_gnn_layers=4, batch_size=8, eval_batch_size=16, lr=1e-5, loss_fn=get_loss, inference_fn=inference_step, dataset=ds, model=model)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "trace = prof.get_trace()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('Module | Self CPU total | CPU total | Number of Calls\\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|-----------|----------------\\n-GAT | | | \\n-dropout | | | \\n--aten::dropout | 142.000us | 142.000us | 1413 \\n--cudaDeviceSynchronize | 5.410ms | 5.410ms | 1413 \\n-act | | | \\n--aten::relu | 252.233ms | 476.470ms | 1413 \\n--aten::clamp_min | 23.247ms | 224.237ms | 1413 \\n--cudaLaunchKernel | 200.990ms | 200.990ms | 1413 \\n--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 1413 \\n--cudaDeviceSynchronize | 3.242ms | 3.242ms | 1413 \\n-convs | | | \\n--0 | | | \\n---aggr_module | | | \\n----aten::view | 1.461ms | 1.461ms | 471 \\n----aten::expand_as | 589.000us | 1.912ms | 471 \\n----aten::expand | 1.001ms | 1.367ms | 471 \\n----aten::as_strided | 428.000us | 428.000us | 1413 \\n----aten::new_zeros | 3.599ms | 180.361ms | 471 \\n----aten::new_empty | 2.758ms | 93.925ms | 471 \\n----aten::empty | 93.399ms | 93.399ms | 471 \\n----aten::zero_ | 1.110ms | 84.988ms | 471 \\n----aten::fill_ | 3.016ms | 83.878ms | 471 \\n----cudaLaunchKernel | 83.662ms | 83.662ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.865ms | 10.726ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.188ms | 1.188ms | 471 \\n---lin | | | \\n----aten::linear | 1.987ms | 211.347ms | 471 \\n----aten::t | 2.331ms | 3.810ms | 471 \\n----aten::transpose | 1.002ms | 1.478ms | 471 \\n----aten::as_strided | 477.000us | 477.000us | 471 \\n----aten::matmul | 1.079ms | 205.550ms | 471 \\n----aten::mm | 107.088ms | 204.471ms | 471 \\n----cudaFree | 5.830ms | 5.830ms | 2 \\n----cudaDeviceGetAttribute | 0.000us | 0.000us | 14 \\n----cudaGetSymbolAddress | 71.000us | 71.000us | 1 \\n----cudaMalloc | 171.000us | 171.000us | 3 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 984.000us | 984.000us | 241 \\n----cudaLaunchKernel | 80.716ms | 80.716ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.609ms | 1.609ms | 471 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 9.611ms | 9.611ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--1 | | | \\n---aggr_module | | | \\n----aten::view | 1.464ms | 1.464ms | 471 \\n----aten::expand_as | 582.000us | 1.875ms | 471 \\n----aten::expand | 988.000us | 1.353ms | 471 \\n----aten::as_strided | 429.000us | 429.000us | 1413 \\n----aten::new_zeros | 2.802ms | 171.042ms | 471 \\n----aten::new_empty | 3.360ms | 98.481ms | 471 \\n----aten::empty | 98.016ms | 98.016ms | 471 \\n----aten::zero_ | 1.045ms | 71.202ms | 471 \\n----aten::fill_ | 2.970ms | 70.157ms | 471 \\n----cudaLaunchKernel | 69.697ms | 69.697ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.770ms | 10.343ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.054ms | 1.054ms | 471 \\n---lin | | | \\n----aten::linear | 6.509ms | 182.224ms | 471 \\n----aten::t | 2.271ms | 3.509ms | 471 \\n----aten::transpose | 760.000us | 1.238ms | 471 \\n----aten::as_strided | 478.000us | 478.000us | 471 \\n----aten::matmul | 967.000us | 177.245ms | 471 \\n----aten::mm | 99.358ms | 176.278ms | 471 \\n----cudaLaunchKernel | 68.871ms | 68.871ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.225ms | 1.225ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 196.000us | 196.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 7.853ms | 7.853ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--2 | | | \\n---aggr_module | | | \\n----aten::view | 1.469ms | 1.469ms | 471 \\n----aten::expand_as | 584.000us | 1.927ms | 471 \\n----aten::expand | 1.040ms | 1.424ms | 471 \\n----aten::as_strided | 461.000us | 461.000us | 1413 \\n----aten::new_zeros | 2.053ms | 164.227ms | 471 \\n----aten::new_empty | 3.461ms | 100.500ms | 471 \\n----aten::empty | 100.034ms | 100.034ms | 471 \\n----aten::zero_ | 1.031ms | 62.406ms | 471 \\n----aten::fill_ | 2.997ms | 61.375ms | 471 \\n----cudaLaunchKernel | 60.833ms | 60.833ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.764ms | 10.296ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.017ms | 1.017ms | 471 \\n---lin | | | \\n----aten::linear | 6.985ms | 167.387ms | 471 \\n----aten::t | 2.268ms | 3.526ms | 471 \\n----aten::transpose | 784.000us | 1.258ms | 471 \\n----aten::as_strided | 474.000us | 474.000us | 471 \\n----aten::matmul | 979.000us | 162.455ms | 471 \\n----aten::mm | 95.898ms | 161.476ms | 471 \\n----cudaLaunchKernel | 59.421ms | 59.421ms | 896 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 98 \\n----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*) | 0.000us | 0.000us | 425 \\n----cudaDeviceSynchronize | 1.170ms | 1.170ms | 471 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 186.000us | 186.000us | 132 \\n----ampere_sgemm_64x32_sliced1x4_tn | 0.000us | 0.000us | 132 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 241 \\n----cudaMemsetAsync | 5.971ms | 5.971ms | 44 \\n----Memset (Device) | 0.000us | 0.000us | 44 \\n--3 | | | \\n---aggr_module | | | \\n----aten::view | 1.455ms | 1.455ms | 471 \\n----aten::expand_as | 626.000us | 1.932ms | 471 \\n----aten::expand | 1.094ms | 1.507ms | 471 \\n----aten::as_strided | 508.000us | 508.000us | 1413 \\n----aten::new_zeros | 2.890ms | 149.597ms | 471 \\n----aten::new_empty | 2.133ms | 89.208ms | 471 \\n----aten::empty | 88.735ms | 88.735ms | 471 \\n----aten::zero_ | 1.005ms | 59.082ms | 471 \\n----aten::fill_ | 2.999ms | 58.077ms | 471 \\n----cudaLaunchKernel | 57.526ms | 57.526ms | 942 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array) | 0.000us | 0.000us | 471 \\n----aten::scatter_add_ | 7.812ms | 10.355ms | 471 \\n----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.048ms | 1.048ms | 471 \\n---lin | | | \\n----aten::linear | 6.715ms | 157.757ms | 471 \\n----aten::t | 2.212ms | 3.499ms | 471 \\n----aten::transpose | 813.000us | 1.287ms | 471 \\n----aten::as_strided | 474.000us | 474.000us | 471 \\n----aten::matmul | 959.000us | 152.888ms | 471 \\n----aten::mm | 92.151ms | 151.929ms | 471 \\n----cudaStreamIsCapturing | 1.000us | 1.000us | 1 \\n----cudaMalloc | 145.000us | 145.000us | 1 \\n----cudaOccupancyMaxActiveBlocksPerMultiprocessor | 1.077ms | 1.077ms | 511 \\n----cudaLaunchKernel | 30.929ms | 30.929ms | 471 \\n----ampere_sgemm_32x32_sliced1x4_tn | 0.000us | 0.000us | 172 \\n----cudaDeviceSynchronize | 6.041ms | 6.041ms | 471 \\n----cudaMemsetAsync | 27.626ms | 27.626ms | 230 \\n----Memset (Device) | 0.000us | 0.000us | 230 \\n----ampere_sgemm_128x32_tn | 0.000us | 0.000us | 214 \\n----ampere_sgemm_128x32_sliced1x4_tn | 0.000us | 0.000us | 83 \\n----ampere_sgemm_128x64_tn | 0.000us | 0.000us | 2 \\n-norms | | | \\n--0 | | | \\n---module | | | \\n----[memory] | 0.000us | 0.000us | 744 \\n----aten::add_ | 70.478ms | 126.577ms | 353 \\n----cudaLaunchKernel | 65.908ms | 65.908ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | -532.000us | 82.041ms | 471 \\n----aten::_batch_norm_impl_index | 4.488ms | 80.580ms | 471 \\n----aten::empty | 28.837ms | 28.837ms | 1413 \\n----aten::native_batch_norm | 12.558ms | 51.947ms | 471 \\n----aten::empty_like | 722.000us | 3.170ms | 471 \\n----aten::empty_strided | 2.523ms | 2.523ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.191ms | 1.191ms | 471 \\n----aten::copy_ | 781.000us | 23.765ms | 118 \\n----cudaMemcpyAsync | 22.984ms | 22.984ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--1 | | | \\n---module | | | \\n----[memory] | 0.000us | 0.000us | 739 \\n----aten::add_ | 81.224ms | 129.447ms | 353 \\n----cudaLaunchKernel | 55.220ms | 55.220ms | 1648 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n----aten::batch_norm | 661.000us | 72.788ms | 471 \\n----aten::_batch_norm_impl_index | 2.810ms | 71.489ms | 471 \\n----aten::empty | 29.551ms | 29.551ms | 1413 \\n----aten::native_batch_norm | 11.271ms | 42.637ms | 471 \\n----aten::empty_like | 667.000us | 3.107ms | 471 \\n----aten::empty_strided | 2.492ms | 2.492ms | 471 \\n----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n----cudaDeviceSynchronize | 1.099ms | 1.099ms | 471 \\n----aten::copy_ | 776.000us | 18.528ms | 118 \\n----cudaMemcpyAsync | 17.752ms | 17.752ms | 118 \\n----Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--2 | | | \\n---module | | | \\n-- -[memory] | 0.000us | 0.000us | 741 \\n-- -aten::add_ | 79.923ms | 123.217ms | 353 \\n-- -cudaLaunchKernel | 50.052ms | 50.052ms | 1648 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array) | 0.000us | 0.000us | 353 \\n-- -aten::batch_norm | 1.028ms | 69.357ms | 471 \\n-- -aten::_batch_norm_impl_index | 2.529ms | 68.094ms | 471 \\n-- -aten::empty | 28.910ms | 28.910ms | 1413 \\n-- -aten::native_batch_norm | 11.237ms | 39.924ms | 471 \\n-- -aten::empty_like | 659.000us | 3.080ms | 471 \\n-- -aten::empty_strided | 2.473ms | 2.473ms | 471 \\n-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float) | 0.000us | 0.000us | 353 \\n-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>) | 0.000us | 0.000us | 353 \\n-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool) | 0.000us | 0.000us | 471 \\n-- -cudaDeviceSynchronize | 1.042ms | 1.042ms | 471 \\n-- -aten::copy_ | 777.000us | 16.076ms | 118 \\n-- -cudaMemcpyAsync | 15.299ms | 15.299ms | 118 \\n-- -Memcpy DtoD (Device -> Device) | 0.000us | 0.000us | 118 \\n-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array) | 0.000us | 0.000us | 118 \\n--3 | | | \\n-_trim | | | \\n',\n", - " ['Module',\n", - " 'Self CPU total',\n", - " 'CPU total',\n", - " 'Self CUDA total',\n", - " 'CUDA total',\n", - " 'Self CPU Mem',\n", - " 'CPU Mem',\n", - " 'Self CUDA Mem',\n", - " 'CUDA Mem',\n", - " 'Number of Calls'],\n", - " {'aten::dropout': [142, 142, 0, 0, 0, 0, 0, 0, 1413],\n", - " 'cudaDeviceSynchronize': [1042, 1042, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::relu': [252233, 476470, 0, 1413, 0, 0, 0, 460271616, 1413],\n", - " 'aten::clamp_min': [23247,\n", - " 224237,\n", - " 1413,\n", - " 1413,\n", - " 0,\n", - " 0,\n", - " 460271616,\n", - " 460271616,\n", - " 1413],\n", - " 'cudaLaunchKernel': [50052, 50052, 0, 0, 0, 0, 0, 0, 1648],\n", - " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", - " 0,\n", - " 1413,\n", - " 1413,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 1413],\n", - " 'aten::view': [1455, 1455, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::expand_as': [626, 1932, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::expand': [1094, 1507, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::as_strided': [474, 474, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::new_zeros': [2890, 149597, 0, 334, 0, 0, -10797056, 731824128, 471],\n", - " 'aten::new_empty': [2133, 89208, 0, 0, 0, 0, 10797056, 731824128, 471],\n", - " 'aten::empty': [28910, 28910, 0, 0, 0, 0, 3858432, 3858432, 1413],\n", - " 'aten::zero_': [1005, 59082, 0, 346, 0, 0, 0, 0, 471],\n", - " 'aten::fill_': [2999, 58077, 346, 346, 0, 0, 0, 0, 471],\n", - " 'void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)': [0,\n", - " 0,\n", - " 346,\n", - " 346,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 471],\n", - " 'aten::scatter_add_': [7812, 10355, 1277, 1277, 0, 0, 0, 0, 471],\n", - " 'void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})': [0,\n", - " 0,\n", - " 1277,\n", - " 1277,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 471],\n", - " 'aten::linear': [6715, 157757, 0, 16142, 0, 0, 21594112, 781910016, 471],\n", - " 'aten::t': [2212, 3499, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::transpose': [813, 1287, 0, 0, 0, 0, 0, 0, 471],\n", - " 'aten::matmul': [959, 152888, 0, 16628, 0, 0, 0, 781910016, 471],\n", - " 'aten::mm': [92151, 151929, 16628, 16628, 0, 0, 781910016, 781910016, 471],\n", - " 'cudaFree': [5830, 5830, 0, 0, 0, 0, 0, 0, 2],\n", - " 'cudaDeviceGetAttribute': [0, 0, 0, 0, 0, 0, 0, 0, 14],\n", - " 'cudaGetSymbolAddress': [71, 71, 0, 0, 0, 0, 0, 0, 1],\n", - " 'cudaMalloc': [145, 145, 0, 0, 0, 0, 0, 0, 1],\n", - " 'cudaOccupancyMaxActiveBlocksPerMultiprocessor': [1077,\n", - " 1077,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 511],\n", - " 'ampere_sgemm_128x32_tn': [0, 0, 9588, 9588, 0, 0, 0, 0, 214],\n", - " 'void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)': [0,\n", - " 0,\n", - " 434,\n", - " 434,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 425],\n", - " 'ampere_sgemm_64x32_sliced1x4_tn': [0, 0, 1419, 1419, 0, 0, 0, 0, 132],\n", - " 'ampere_sgemm_32x32_sliced1x4_tn': [0, 0, 4197, 4197, 0, 0, 0, 0, 172],\n", - " 'cudaMemsetAsync': [27626, 27626, 0, 0, 0, 0, 0, 0, 230],\n", - " 'Memset (Device)': [0, 0, 0, 0, 0, 0, 0, 0, 230],\n", - " 'cudaStreamIsCapturing': [1, 1, 0, 0, 0, 0, 0, 0, 1],\n", - " 'ampere_sgemm_128x32_sliced1x4_tn': [0, 0, 2739, 2739, 0, 0, 0, 0, 83],\n", - " 'ampere_sgemm_128x64_tn': [0, 0, 104, 104, 0, 0, 0, 0, 2],\n", - " '[memory]': [0, 0, 0, 0, 0, 0, -143360, -143360, 741],\n", - " 'aten::add_': [79923, 123217, 353, 353, 0, 0, 0, 0, 353],\n", - " 'void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)': [0,\n", - " 0,\n", - " 353,\n", - " 353,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 353],\n", - " 'aten::batch_norm': [1028, 69357, 0, 2562, 0, 0, -1417216, 156459008, 471],\n", - " 'aten::_batch_norm_impl_index': [2529,\n", - " 68094,\n", - " 0,\n", - " 2560,\n", - " 0,\n", - " 0,\n", - " 1871872,\n", - " 157282304,\n", - " 471],\n", - " 'aten::native_batch_norm': [11237,\n", - " 39924,\n", - " 2474,\n", - " 2592,\n", - " 0,\n", - " 0,\n", - " -3280896,\n", - " 157282304,\n", - " 471],\n", - " 'aten::empty_like': [659, 3080, 0, 0, 0, 0, 3280896, 153423872, 471],\n", - " 'aten::empty_strided': [2473, 2473, 0, 0, 0, 0, 153423872, 153423872, 471],\n", - " 'void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)': [0,\n", - " 0,\n", - " 1061,\n", - " 1061,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 353],\n", - " 'void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)': [0,\n", - " 0,\n", - " 353,\n", - " 353,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 353],\n", - " 'void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)': [0,\n", - " 0,\n", - " 942,\n", - " 942,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 471],\n", - " 'aten::copy_': [777, 16076, 118, 118, 0, 0, 0, 0, 118],\n", - " 'cudaMemcpyAsync': [15299, 15299, 0, 0, 0, 0, 0, 0, 118],\n", - " 'Memcpy DtoD (Device -> Device)': [0, 0, 118, 118, 0, 0, 0, 0, 118],\n", - " 'void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)': [0,\n", - " 0,\n", - " 118,\n", - " 118,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 118]},\n", - " ['-dropout--aten::dropout',\n", - " '-dropout--cudaDeviceSynchronize',\n", - " '-act--aten::relu',\n", - " '-act--aten::clamp_min',\n", - " '-act--cudaLaunchKernel',\n", - " '-act--void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::launch_clamp_scalar(at::TensorIteratorBase&, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::{lambda()#1}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", - " '-act--cudaDeviceSynchronize',\n", - " '-convs--0---aggr_module----aten::view',\n", - " '-convs--0---aggr_module----aten::expand_as',\n", - " '-convs--0---aggr_module----aten::expand',\n", - " '-convs--0---aggr_module----aten::as_strided',\n", - " '-convs--0---aggr_module----aten::new_zeros',\n", - " '-convs--0---aggr_module----aten::new_empty',\n", - " '-convs--0---aggr_module----aten::empty',\n", - " '-convs--0---aggr_module----aten::zero_',\n", - " '-convs--0---aggr_module----aten::fill_',\n", - " '-convs--0---aggr_module----cudaLaunchKernel',\n", - " '-convs--0---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", - " '-convs--0---aggr_module----aten::scatter_add_',\n", - " '-convs--0---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", - " '-convs--0---aggr_module----cudaDeviceSynchronize',\n", - " '-convs--0---lin----aten::linear',\n", - " '-convs--0---lin----aten::t',\n", - " '-convs--0---lin----aten::transpose',\n", - " '-convs--0---lin----aten::as_strided',\n", - " '-convs--0---lin----aten::matmul',\n", - " '-convs--0---lin----aten::mm',\n", - " '-convs--0---lin----cudaFree',\n", - " '-convs--0---lin----cudaDeviceGetAttribute',\n", - " '-convs--0---lin----cudaGetSymbolAddress',\n", - " '-convs--0---lin----cudaMalloc',\n", - " '-convs--0---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", - " '-convs--0---lin----cudaLaunchKernel',\n", - " '-convs--0---lin----ampere_sgemm_128x32_tn',\n", - " '-convs--0---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", - " '-convs--0---lin----cudaDeviceSynchronize',\n", - " '-convs--0---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", - " '-convs--0---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", - " '-convs--0---lin----cudaMemsetAsync',\n", - " '-convs--0---lin----Memset (Device)',\n", - " '-convs--1---aggr_module----aten::view',\n", - " '-convs--1---aggr_module----aten::expand_as',\n", - " '-convs--1---aggr_module----aten::expand',\n", - " '-convs--1---aggr_module----aten::as_strided',\n", - " '-convs--1---aggr_module----aten::new_zeros',\n", - " '-convs--1---aggr_module----aten::new_empty',\n", - " '-convs--1---aggr_module----aten::empty',\n", - " '-convs--1---aggr_module----aten::zero_',\n", - " '-convs--1---aggr_module----aten::fill_',\n", - " '-convs--1---aggr_module----cudaLaunchKernel',\n", - " '-convs--1---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", - " '-convs--1---aggr_module----aten::scatter_add_',\n", - " '-convs--1---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", - " '-convs--1---aggr_module----cudaDeviceSynchronize',\n", - " '-convs--1---lin----aten::linear',\n", - " '-convs--1---lin----aten::t',\n", - " '-convs--1---lin----aten::transpose',\n", - " '-convs--1---lin----aten::as_strided',\n", - " '-convs--1---lin----aten::matmul',\n", - " '-convs--1---lin----aten::mm',\n", - " '-convs--1---lin----cudaLaunchKernel',\n", - " '-convs--1---lin----ampere_sgemm_128x32_tn',\n", - " '-convs--1---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", - " '-convs--1---lin----cudaDeviceSynchronize',\n", - " '-convs--1---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", - " '-convs--1---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", - " '-convs--1---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", - " '-convs--1---lin----cudaMemsetAsync',\n", - " '-convs--1---lin----Memset (Device)',\n", - " '-convs--2---aggr_module----aten::view',\n", - " '-convs--2---aggr_module----aten::expand_as',\n", - " '-convs--2---aggr_module----aten::expand',\n", - " '-convs--2---aggr_module----aten::as_strided',\n", - " '-convs--2---aggr_module----aten::new_zeros',\n", - " '-convs--2---aggr_module----aten::new_empty',\n", - " '-convs--2---aggr_module----aten::empty',\n", - " '-convs--2---aggr_module----aten::zero_',\n", - " '-convs--2---aggr_module----aten::fill_',\n", - " '-convs--2---aggr_module----cudaLaunchKernel',\n", - " '-convs--2---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", - " '-convs--2---aggr_module----aten::scatter_add_',\n", - " '-convs--2---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", - " '-convs--2---aggr_module----cudaDeviceSynchronize',\n", - " '-convs--2---lin----aten::linear',\n", - " '-convs--2---lin----aten::t',\n", - " '-convs--2---lin----aten::transpose',\n", - " '-convs--2---lin----aten::as_strided',\n", - " '-convs--2---lin----aten::matmul',\n", - " '-convs--2---lin----aten::mm',\n", - " '-convs--2---lin----cudaLaunchKernel',\n", - " '-convs--2---lin----ampere_sgemm_128x32_tn',\n", - " '-convs--2---lin----void splitKreduce_kernel<32, 16, int, float, float, float, float, true, false, false>(cublasSplitKParams, float const*, float const*, float*, float const*, float const*, float const*, float const*, float*, void*, long, float*, int*)',\n", - " '-convs--2---lin----cudaDeviceSynchronize',\n", - " '-convs--2---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", - " '-convs--2---lin----ampere_sgemm_64x32_sliced1x4_tn',\n", - " '-convs--2---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", - " '-convs--2---lin----cudaMemsetAsync',\n", - " '-convs--2---lin----Memset (Device)',\n", - " '-convs--3---aggr_module----aten::view',\n", - " '-convs--3---aggr_module----aten::expand_as',\n", - " '-convs--3---aggr_module----aten::expand',\n", - " '-convs--3---aggr_module----aten::as_strided',\n", - " '-convs--3---aggr_module----aten::new_zeros',\n", - " '-convs--3---aggr_module----aten::new_empty',\n", - " '-convs--3---aggr_module----aten::empty',\n", - " '-convs--3---aggr_module----aten::zero_',\n", - " '-convs--3---aggr_module----aten::fill_',\n", - " '-convs--3---aggr_module----cudaLaunchKernel',\n", - " '-convs--3---aggr_module----void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor, at::detail::Array >(int, at::native::FillFunctor, at::detail::Array)',\n", - " '-convs--3---aggr_module----aten::scatter_add_',\n", - " '-convs--3---aggr_module----void at::native::_scatter_gather_elementwise_kernel<128, 4, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1}>(int, at::native::_cuda_scatter_gather_internal_kernel::operator()(at::TensorIterator&, long, long, long, at::native::ReduceAdd const&)::{lambda(int)#1})',\n", - " '-convs--3---aggr_module----cudaDeviceSynchronize',\n", - " '-convs--3---lin----aten::linear',\n", - " '-convs--3---lin----aten::t',\n", - " '-convs--3---lin----aten::transpose',\n", - " '-convs--3---lin----aten::as_strided',\n", - " '-convs--3---lin----aten::matmul',\n", - " '-convs--3---lin----aten::mm',\n", - " '-convs--3---lin----cudaStreamIsCapturing',\n", - " '-convs--3---lin----cudaMalloc',\n", - " '-convs--3---lin----cudaOccupancyMaxActiveBlocksPerMultiprocessor',\n", - " '-convs--3---lin----cudaLaunchKernel',\n", - " '-convs--3---lin----ampere_sgemm_32x32_sliced1x4_tn',\n", - " '-convs--3---lin----cudaDeviceSynchronize',\n", - " '-convs--3---lin----cudaMemsetAsync',\n", - " '-convs--3---lin----Memset (Device)',\n", - " '-convs--3---lin----ampere_sgemm_128x32_tn',\n", - " '-convs--3---lin----ampere_sgemm_128x32_sliced1x4_tn',\n", - " '-convs--3---lin----ampere_sgemm_128x64_tn',\n", - " '-norms--0---module----[memory]',\n", - " '-norms--0---module----aten::add_',\n", - " '-norms--0---module----cudaLaunchKernel',\n", - " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", - " '-norms--0---module----aten::batch_norm',\n", - " '-norms--0---module----aten::_batch_norm_impl_index',\n", - " '-norms--0---module----aten::empty',\n", - " '-norms--0---module----aten::native_batch_norm',\n", - " '-norms--0---module----aten::empty_like',\n", - " '-norms--0---module----aten::empty_strided',\n", - " '-norms--0---module----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", - " '-norms--0---module----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", - " '-norms--0---module----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", - " '-norms--0---module----cudaDeviceSynchronize',\n", - " '-norms--0---module----aten::copy_',\n", - " '-norms--0---module----cudaMemcpyAsync',\n", - " '-norms--0---module----Memcpy DtoD (Device -> Device)',\n", - " '-norms--0---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", - " '-norms--1---module----[memory]',\n", - " '-norms--1---module----aten::add_',\n", - " '-norms--1---module----cudaLaunchKernel',\n", - " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", - " '-norms--1---module----aten::batch_norm',\n", - " '-norms--1---module----aten::_batch_norm_impl_index',\n", - " '-norms--1---module----aten::empty',\n", - " '-norms--1---module----aten::native_batch_norm',\n", - " '-norms--1---module----aten::empty_like',\n", - " '-norms--1---module----aten::empty_strided',\n", - " '-norms--1---module----void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", - " '-norms--1---module----void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", - " '-norms--1---module----void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", - " '-norms--1---module----cudaDeviceSynchronize',\n", - " '-norms--1---module----aten::copy_',\n", - " '-norms--1---module----cudaMemcpyAsync',\n", - " '-norms--1---module----Memcpy DtoD (Device -> Device)',\n", - " '-norms--1---module----void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)',\n", - " '-norms-- -[memory]',\n", - " '-norms-- -aten::add_',\n", - " '-norms-- -cudaLaunchKernel',\n", - " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctorOnSelf_add, at::detail::Array >(int, at::native::CUDAFunctorOnSelf_add, at::detail::Array)',\n", - " '-norms-- -aten::batch_norm',\n", - " '-norms-- -aten::_batch_norm_impl_index',\n", - " '-norms-- -aten::empty',\n", - " '-norms-- -aten::native_batch_norm',\n", - " '-norms-- -aten::empty_like',\n", - " '-norms-- -aten::empty_strided',\n", - " '-norms-- -void at::native::batch_norm_collect_statistics_channels_last_kernel(float const*, float*, float*, float volatile*, int*, int, int, float)',\n", - " '-norms-- -void at::native::(anonymous namespace)::unrolled_elementwise_kernel_for_multi_outputs<3, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int> >(int, at::native::(anonymous namespace)::batch_norm_update_stats_and_invert(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, double, double, long)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float, float, float, float)#1}, at::detail::Array, TrivialOffsetCalculator<4, unsigned int>, TrivialOffsetCalculator<3, unsigned int>)',\n", - " '-norms-- -void at::native::batch_norm_transform_input_channels_last_kernel(float const*, float const*, float const*, float const*, float const*, float const*, float*, int, int, bool)',\n", - " '-norms-- -cudaDeviceSynchronize',\n", - " '-norms-- -aten::copy_',\n", - " '-norms-- -cudaMemcpyAsync',\n", - " '-norms-- -Memcpy DtoD (Device -> Device)',\n", - " '-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array)'],\n", - " [['142.000us',\n", - " '142.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['5.410ms',\n", - " '5.410ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['252.233ms',\n", - " '476.470ms',\n", - " '0.000us',\n", - " '1.413ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '438.95 Mb',\n", - " '1413'],\n", - " ['23.247ms',\n", - " '224.237ms',\n", - " '1.413ms',\n", - " '1.413ms',\n", - " '0 b',\n", - " '0 b',\n", - " '438.95 Mb',\n", - " '438.95 Mb',\n", - " '1413'],\n", - " ['200.990ms',\n", - " '200.990ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.413ms',\n", - " '1.413ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['3.242ms',\n", - " '3.242ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['1.461ms',\n", - " '1.461ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['589.000us',\n", - " '1.912ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.001ms',\n", - " '1.367ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['428.000us',\n", - " '428.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['3.599ms',\n", - " '180.361ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-3.38 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.758ms',\n", - " '93.925ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.38 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['93.399ms',\n", - " '93.399ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['1.110ms',\n", - " '84.988ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['3.016ms',\n", - " '83.878ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['83.662ms',\n", - " '83.662ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '942'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['7.865ms',\n", - " '10.726ms',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.188ms',\n", - " '1.188ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.987ms',\n", - " '211.347ms',\n", - " '0.000us',\n", - " '6.005ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '154.44 Mb',\n", - " '471'],\n", - " ['2.331ms',\n", - " '3.810ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.002ms',\n", - " '1.478ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['477.000us',\n", - " '477.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.079ms',\n", - " '205.550ms',\n", - " '0.000us',\n", - " '6.005ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '154.44 Mb',\n", - " '471'],\n", - " ['107.088ms',\n", - " '204.471ms',\n", - " '6.005ms',\n", - " '6.005ms',\n", - " '0 b',\n", - " '0 b',\n", - " '154.44 Mb',\n", - " '154.44 Mb',\n", - " '471'],\n", - " ['5.830ms',\n", - " '5.830ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '2'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '14'],\n", - " ['71.000us',\n", - " '71.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1'],\n", - " ['171.000us',\n", - " '171.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '3'],\n", - " ['984.000us',\n", - " '984.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '241'],\n", - " ['80.716ms',\n", - " '80.716ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '896'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '813.000us',\n", - " '813.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '98'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '500.000us',\n", - " '500.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '425'],\n", - " ['1.609ms',\n", - " '1.609ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.605ms',\n", - " '1.605ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '132'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '3.087ms',\n", - " '3.087ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '241'],\n", - " ['9.611ms',\n", - " '9.611ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['1.464ms',\n", - " '1.464ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['582.000us',\n", - " '1.875ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['988.000us',\n", - " '1.353ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['429.000us',\n", - " '429.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['2.802ms',\n", - " '171.042ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-4.36 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['3.360ms',\n", - " '98.481ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '4.36 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['98.016ms',\n", - " '98.016ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['1.045ms',\n", - " '71.202ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['2.970ms',\n", - " '70.157ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['69.697ms',\n", - " '69.697ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '942'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['7.770ms',\n", - " '10.343ms',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.054ms',\n", - " '1.054ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['6.509ms',\n", - " '182.224ms',\n", - " '0.000us',\n", - " '5.265ms',\n", - " '0 b',\n", - " '0 b',\n", - " '3.70 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.271ms',\n", - " '3.509ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['760.000us',\n", - " '1.238ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['478.000us',\n", - " '478.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['967.000us',\n", - " '177.245ms',\n", - " '0.000us',\n", - " '5.410ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['99.358ms',\n", - " '176.278ms',\n", - " '5.410ms',\n", - " '5.410ms',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['68.871ms',\n", - " '68.871ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '896'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '688.000us',\n", - " '688.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '98'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '434.000us',\n", - " '434.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '425'],\n", - " ['1.225ms',\n", - " '1.225ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['196.000us',\n", - " '196.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '132'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.407ms',\n", - " '1.407ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '132'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '2.881ms',\n", - " '2.881ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '241'],\n", - " ['7.853ms',\n", - " '7.853ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['1.469ms',\n", - " '1.469ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['584.000us',\n", - " '1.927ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.040ms',\n", - " '1.424ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['461.000us',\n", - " '461.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['2.053ms',\n", - " '164.227ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-4.03 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['3.461ms',\n", - " '100.500ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '4.03 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['100.034ms',\n", - " '100.034ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['1.031ms',\n", - " '62.406ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['2.997ms',\n", - " '61.375ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['60.833ms',\n", - " '60.833ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '942'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['7.764ms',\n", - " '10.296ms',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.017ms',\n", - " '1.017ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['6.985ms',\n", - " '167.387ms',\n", - " '0.000us',\n", - " '5.307ms',\n", - " '0 b',\n", - " '0 b',\n", - " '4.06 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.268ms',\n", - " '3.526ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['784.000us',\n", - " '1.258ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['474.000us',\n", - " '474.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['979.000us',\n", - " '162.455ms',\n", - " '0.000us',\n", - " '5.470ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['95.898ms',\n", - " '161.476ms',\n", - " '5.470ms',\n", - " '5.470ms',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['59.421ms',\n", - " '59.421ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '896'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '688.000us',\n", - " '688.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '98'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '434.000us',\n", - " '434.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '425'],\n", - " ['1.170ms',\n", - " '1.170ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['186.000us',\n", - " '186.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '132'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.419ms',\n", - " '1.419ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '132'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '2.929ms',\n", - " '2.929ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '241'],\n", - " ['5.971ms',\n", - " '5.971ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '44'],\n", - " ['1.455ms',\n", - " '1.455ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['626.000us',\n", - " '1.932ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.094ms',\n", - " '1.507ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['508.000us',\n", - " '508.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1413'],\n", - " ['2.890ms',\n", - " '149.597ms',\n", - " '0.000us',\n", - " '334.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-10.30 Mb',\n", - " '697.92 Mb',\n", - " '471'],\n", - " ['2.133ms',\n", - " '89.208ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '10.30 Mb',\n", - " '697.92 Mb',\n", - " '471'],\n", - " ['88.735ms',\n", - " '88.735ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '697.92 Mb',\n", - " '697.92 Mb',\n", - " '471'],\n", - " ['1.005ms',\n", - " '59.082ms',\n", - " '0.000us',\n", - " '346.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['2.999ms',\n", - " '58.077ms',\n", - " '346.000us',\n", - " '346.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['57.526ms',\n", - " '57.526ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '942'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '346.000us',\n", - " '346.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['7.812ms',\n", - " '10.355ms',\n", - " '1.277ms',\n", - " '1.277ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.277ms',\n", - " '1.277ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.048ms',\n", - " '1.048ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['6.715ms',\n", - " '157.757ms',\n", - " '0.000us',\n", - " '16.142ms',\n", - " '0 b',\n", - " '0 b',\n", - " '20.59 Mb',\n", - " '745.69 Mb',\n", - " '471'],\n", - " ['2.212ms',\n", - " '3.499ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['813.000us',\n", - " '1.287ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['474.000us',\n", - " '474.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['959.000us',\n", - " '152.888ms',\n", - " '0.000us',\n", - " '16.628ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '745.69 Mb',\n", - " '471'],\n", - " ['92.151ms',\n", - " '151.929ms',\n", - " '16.628ms',\n", - " '16.628ms',\n", - " '0 b',\n", - " '0 b',\n", - " '745.69 Mb',\n", - " '745.69 Mb',\n", - " '471'],\n", - " ['1.000us',\n", - " '1.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1'],\n", - " ['145.000us',\n", - " '145.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1'],\n", - " ['1.077ms',\n", - " '1.077ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '511'],\n", - " ['30.929ms',\n", - " '30.929ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '4.197ms',\n", - " '4.197ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '172'],\n", - " ['6.041ms',\n", - " '6.041ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['27.626ms',\n", - " '27.626ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '230'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '230'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '9.588ms',\n", - " '9.588ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '214'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '2.739ms',\n", - " '2.739ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '83'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '104.000us',\n", - " '104.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '2'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-152.00 Kb',\n", - " '-152.00 Kb',\n", - " '744'],\n", - " ['70.478ms',\n", - " '126.577ms',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['65.908ms',\n", - " '65.908ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1648'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['-532.000us',\n", - " '82.041ms',\n", - " '0.000us',\n", - " '2.580ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-3.91 Mb',\n", - " '149.22 Mb',\n", - " '471'],\n", - " ['4.488ms',\n", - " '80.580ms',\n", - " '0.000us',\n", - " '2.540ms',\n", - " '0 b',\n", - " '0 b',\n", - " '3.60 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['28.837ms',\n", - " '28.837ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.68 Mb',\n", - " '3.68 Mb',\n", - " '1413'],\n", - " ['12.558ms',\n", - " '51.947ms',\n", - " '2.474ms',\n", - " '2.592ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-5.31 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['722.000us',\n", - " '3.170ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '5.31 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.523ms',\n", - " '2.523ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.061ms',\n", - " '1.061ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.191ms',\n", - " '1.191ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['781.000us',\n", - " '23.765ms',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['22.984ms',\n", - " '22.984ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-132.00 Kb',\n", - " '-132.00 Kb',\n", - " '739'],\n", - " ['81.224ms',\n", - " '129.447ms',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['55.220ms',\n", - " '55.220ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1648'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['661.000us',\n", - " '72.788ms',\n", - " '0.000us',\n", - " '2.571ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-2.89 Mb',\n", - " '149.20 Mb',\n", - " '471'],\n", - " ['2.810ms',\n", - " '71.489ms',\n", - " '0.000us',\n", - " '2.533ms',\n", - " '0 b',\n", - " '0 b',\n", - " '2.98 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['29.551ms',\n", - " '29.551ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.68 Mb',\n", - " '3.68 Mb',\n", - " '1413'],\n", - " ['11.271ms',\n", - " '42.637ms',\n", - " '2.471ms',\n", - " '2.589ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-3.47 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['667.000us',\n", - " '3.107ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.47 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.492ms',\n", - " '2.492ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.058ms',\n", - " '1.058ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.099ms',\n", - " '1.099ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['776.000us',\n", - " '18.528ms',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['17.752ms',\n", - " '17.752ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '-140.00 Kb',\n", - " '-140.00 Kb',\n", - " '741'],\n", - " ['79.923ms',\n", - " '123.217ms',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['50.052ms',\n", - " '50.052ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '1648'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['1.028ms',\n", - " '69.357ms',\n", - " '0.000us',\n", - " '2.562ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-1.35 Mb',\n", - " '149.21 Mb',\n", - " '471'],\n", - " ['2.529ms',\n", - " '68.094ms',\n", - " '0.000us',\n", - " '2.560ms',\n", - " '0 b',\n", - " '0 b',\n", - " '1.79 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['28.910ms',\n", - " '28.910ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.68 Mb',\n", - " '3.68 Mb',\n", - " '1413'],\n", - " ['11.237ms',\n", - " '39.924ms',\n", - " '2.474ms',\n", - " '2.592ms',\n", - " '0 b',\n", - " '0 b',\n", - " '-3.13 Mb',\n", - " '150.00 Mb',\n", - " '471'],\n", - " ['659.000us',\n", - " '3.080ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '3.13 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['2.473ms',\n", - " '2.473ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '146.32 Mb',\n", - " '146.32 Mb',\n", - " '471'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '1.061ms',\n", - " '1.061ms',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '353.000us',\n", - " '353.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '353'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '942.000us',\n", - " '942.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['1.042ms',\n", - " '1.042ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '471'],\n", - " ['777.000us',\n", - " '16.076ms',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['15.299ms',\n", - " '15.299ms',\n", - " '0.000us',\n", - " '0.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118'],\n", - " ['0.000us',\n", - " '0.000us',\n", - " '118.000us',\n", - " '118.000us',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '0 b',\n", - " '118']])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trace" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "with open('trace.txt', 'w') as f:\n", - " f.write(trace[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(trace[1])" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Module',\n", - " 'Self CPU total',\n", - " 'CPU total',\n", - " 'Self CUDA total',\n", - " 'CUDA total',\n", - " 'Self CPU Mem',\n", - " 'CPU Mem',\n", - " 'Self CUDA Mem',\n", - " 'CUDA Mem',\n", - " 'Number of Calls']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trace[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(trace[4], columns=trace[1][1:], index=trace[3])" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Self CPU totalCPU totalSelf CUDA totalCUDA totalSelf CPU MemCPU MemSelf CUDA MemCUDA MemNumber of Calls
-dropout--aten::dropout142.000us142.000us0.000us0.000us0 b0 b0 b0 b1413
-dropout--cudaDeviceSynchronize5.410ms5.410ms0.000us0.000us0 b0 b0 b0 b1413
-act--aten::relu252.233ms476.470ms0.000us1.413ms0 b0 b0 b438.95 Mb1413
-act--aten::clamp_min23.247ms224.237ms1.413ms1.413ms0 b0 b438.95 Mb438.95 Mb1413
-act--cudaLaunchKernel200.990ms200.990ms0.000us0.000us0 b0 b0 b0 b1413
..............................
-norms-- -cudaDeviceSynchronize1.042ms1.042ms0.000us0.000us0 b0 b0 b0 b471
-norms-- -aten::copy_777.000us16.076ms118.000us118.000us0 b0 b0 b0 b118
-norms-- -cudaMemcpyAsync15.299ms15.299ms0.000us0.000us0 b0 b0 b0 b118
-norms-- -Memcpy DtoD (Device -> Device)0.000us0.000us118.000us118.000us0 b0 b0 b0 b118
-norms-- -void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array<char*, 2> >(int, at::native::(anonymous namespace)::batch_norm_calc_invstd(at::Tensor const&, at::Tensor const&, double)::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const::{lambda(float)#1}, at::detail::Array<char*, 2>)0.000us0.000us118.000us118.000us0 b0 b0 b0 b118
\n", - "

183 rows × 9 columns

\n", - "
" - ], - "text/plain": [ - " Self CPU total CPU total \\\n", - "-dropout--aten::dropout 142.000us 142.000us \n", - "-dropout--cudaDeviceSynchronize 5.410ms 5.410ms \n", - "-act--aten::relu 252.233ms 476.470ms \n", - "-act--aten::clamp_min 23.247ms 224.237ms \n", - "-act--cudaLaunchKernel 200.990ms 200.990ms \n", - "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 1.042ms 1.042ms \n", - "-norms-- -aten::copy_ 777.000us 16.076ms \n", - "-norms-- -cudaMemcpyAsync 15.299ms 15.299ms \n", - "-norms-- -Memcpy DtoD (Device -> Device) 0.000us 0.000us \n", - "-norms-- -void at::native::vectorized_elementwi... 0.000us 0.000us \n", - "\n", - " Self CUDA total CUDA total \\\n", - "-dropout--aten::dropout 0.000us 0.000us \n", - "-dropout--cudaDeviceSynchronize 0.000us 0.000us \n", - "-act--aten::relu 0.000us 1.413ms \n", - "-act--aten::clamp_min 1.413ms 1.413ms \n", - "-act--cudaLaunchKernel 0.000us 0.000us \n", - "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 0.000us 0.000us \n", - "-norms-- -aten::copy_ 118.000us 118.000us \n", - "-norms-- -cudaMemcpyAsync 0.000us 0.000us \n", - "-norms-- -Memcpy DtoD (Device -> Device) 118.000us 118.000us \n", - "-norms-- -void at::native::vectorized_elementwi... 118.000us 118.000us \n", - "\n", - " Self CPU Mem CPU Mem \\\n", - "-dropout--aten::dropout 0 b 0 b \n", - "-dropout--cudaDeviceSynchronize 0 b 0 b \n", - "-act--aten::relu 0 b 0 b \n", - "-act--aten::clamp_min 0 b 0 b \n", - "-act--cudaLaunchKernel 0 b 0 b \n", - "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", - "-norms-- -aten::copy_ 0 b 0 b \n", - "-norms-- -cudaMemcpyAsync 0 b 0 b \n", - "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", - "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", - "\n", - " Self CUDA Mem CUDA Mem \\\n", - "-dropout--aten::dropout 0 b 0 b \n", - "-dropout--cudaDeviceSynchronize 0 b 0 b \n", - "-act--aten::relu 0 b 438.95 Mb \n", - "-act--aten::clamp_min 438.95 Mb 438.95 Mb \n", - "-act--cudaLaunchKernel 0 b 0 b \n", - "... ... ... \n", - "-norms-- -cudaDeviceSynchronize 0 b 0 b \n", - "-norms-- -aten::copy_ 0 b 0 b \n", - "-norms-- -cudaMemcpyAsync 0 b 0 b \n", - "-norms-- -Memcpy DtoD (Device -> Device) 0 b 0 b \n", - "-norms-- -void at::native::vectorized_elementwi... 0 b 0 b \n", - "\n", - " Number of Calls \n", - "-dropout--aten::dropout 1413 \n", - "-dropout--cudaDeviceSynchronize 1413 \n", - "-act--aten::relu 1413 \n", - "-act--aten::clamp_min 1413 \n", - "-act--cudaLaunchKernel 1413 \n", - "... ... \n", - "-norms-- -cudaDeviceSynchronize 471 \n", - "-norms-- -aten::copy_ 118 \n", - "-norms-- -cudaMemcpyAsync 118 \n", - "-norms-- -Memcpy DtoD (Device -> Device) 118 \n", - "-norms-- -void at::native::vectorized_elementwi... 118 \n", - "\n", - "[183 rows x 9 columns]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('llm_to_use', LLM(\n", - " (llm): LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32001, 2048)\n", - " (layers): ModuleList(\n", - " (0-21): 22 x LlamaDecoderLayer(\n", - " (self_attn): LlamaSdpaAttention(\n", - " (q_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", - " (k_proj): Linear(in_features=2048, out_features=256, bias=False)\n", - " (v_proj): Linear(in_features=2048, out_features=256, bias=False)\n", - " (o_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", - " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", - " (up_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", - " (down_proj): Linear(in_features=5632, out_features=2048, bias=False)\n", - " (act_fn): SiLU()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", - " )\n", - " )\n", - " (norm): LlamaRMSNorm()\n", - " )\n", - " (lm_head): Linear(in_features=2048, out_features=32001, bias=False)\n", - " )\n", - " (word_embedding): Embedding(32001, 2048)\n", - "))\n", - "('llm_generator', LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32001, 2048)\n", - " (layers): ModuleList(\n", - " (0-21): 22 x LlamaDecoderLayer(\n", - " (self_attn): LlamaSdpaAttention(\n", - " (q_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", - " (k_proj): Linear(in_features=2048, out_features=256, bias=False)\n", - " (v_proj): Linear(in_features=2048, out_features=256, bias=False)\n", - " (o_proj): Linear(in_features=2048, out_features=2048, bias=False)\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", - " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", - " (up_proj): Linear(in_features=2048, out_features=5632, bias=False)\n", - " (down_proj): Linear(in_features=5632, out_features=2048, bias=False)\n", - " (act_fn): SiLU()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", - " )\n", - " )\n", - " (norm): LlamaRMSNorm()\n", - " )\n", - " (lm_head): Linear(in_features=2048, out_features=32001, bias=False)\n", - "))\n", - "('graph_encoder', GAT(1024, 1024, num_layers=4))\n", - "('projector', Sequential(\n", - " (0): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (1): Sigmoid()\n", - " (2): Linear(in_features=1024, out_features=2048, bias=True)\n", - "))\n", - "('word_embedding', Embedding(32001, 2048))\n" - ] - } - ], - "source": [ - "for c in model.named_children():\n", - " print(c)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/test_rag.ipynb b/examples/llm_plus_gnn/test_rag.ipynb deleted file mode 100644 index 286f7b495499..000000000000 --- a/examples/llm_plus_gnn/test_rag.ipynb +++ /dev/null @@ -1,321 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from profiling_utils import create_remote_backend_from_triplets\n", - "from rag_feature_store import SentenceTransformerFeatureStore\n", - "from rag_graph_store import NeighborSamplingRAGGraphStore\n", - "from torch_geometric.loader import RAGQueryLoader\n", - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", - "from torch_geometric.data import get_features_for_triplets_groups, Data\n", - "from itertools import chain\n", - "import torch\n", - "from typing import Tuple" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading graph...\n", - "Encoding questions...\n", - "Retrieving subgraphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "10it [00:00, 58.23it/s]00:00 Tuple[Data, str]:\n", - " q_emb = model.encode(query)\n", - " textual_nodes = ds.textual_nodes.iloc[graph[\"node_idx\"]].reset_index()\n", - " textual_edges = ds.textual_edges.iloc[graph[\"edge_idx\"]].reset_index()\n", - " out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e)\n", - " out_graph[\"desc\"] = desc\n", - " return graph" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={\"k_nodes\": 10}, seed_edges_kwargs={\"k_edges\": 10}, sampler_kwargs={\"num_neighbors\": [40]*10}, local_filter=apply_retrieval_via_pcst)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.5534146021759078 0.0007042666823170373 0.6\n", - "0.5555118626294403 0.00017691808692575338 0.75\n", - "0.5525756979944947 0.0001757881167233095 0.42857142857142855\n", - "0.5580547909293485 0.00011862396204033214 1.0\n", - "0.5605977192292568 5.966587112171838e-05 0.3333333333333333\n", - "0.5426923581072225 0.00017200848575196377 0.3333333333333333\n", - "0.5530475815965396 0.00011729517330361856 1.0\n", - "0.553126228863547 0.0002933239469670304 0.5\n", - "0.5531000131078778 0.00029337557941676935 0.35714285714285715\n", - "0.5271464149954123 5.545389009038984e-05 0.16666666666666666\n" - ] - } - ], - "source": [ - "for subg, gt in zip((query_loader.query(q) for q in questions), ds):\n", - " print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/wikidata.ipynb b/examples/llm_plus_gnn/wikidata.ipynb deleted file mode 100644 index a9e2c9b7bbc3..000000000000 --- a/examples/llm_plus_gnn/wikidata.ipynb +++ /dev/null @@ -1,30776 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import tqdm\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: Download dataset from this notebook instead of from outside it" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "lines = []\n", - "with open(\"/home/zaristei/repos/musique/raw_data/2wikimultihopqa_train_20k.jsonl\") as f:\n", - " for line in f:\n", - " lines.append(json.loads(line))\n", - "df_train = pd.DataFrame(lines)\n", - "df_train['is_train'] = True\n", - "lines = []\n", - "with open(\"/home/zaristei/repos/musique/raw_data/2wikimultihopqa_dev_20k.jsonl\") as f:\n", - " for line in f:\n", - " lines.append(json.loads(line))\n", - "df_test = pd.DataFrame(lines)\n", - "df_test['is_train'] = False\n", - "df = pd.concat([df_train, df_test])" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "is_train\n", - "True 20000\n", - "False 12576\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df['is_train'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "df['type'] = df['other_info'].apply(lambda row: row['type'])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "type\n", - "compositional 14339\n", - "comparison 9318\n", - "bridge_comparison 6834\n", - "inference 2085\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df['type'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "df['graph'] = df['other_info'].apply(lambda row: row['evidences_id'])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "df['graph_size'] = df['graph'].apply(lambda row: len(row))" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "graph_size\n", - "2 21390\n", - "0 8135\n", - "4 2540\n", - "5 328\n", - "3 183\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df['graph_size'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "df = df[df['graph_size'] > 0]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "is_train\n", - "True 16377\n", - "False 8064\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df['is_train'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['dataset', 'id', 'question_text', 'answer_text', 'answerable',\n", - " 'other_info', 'answer_aliases_texts', 'contexts',\n", - " 'decomposed_question_text', 'decomposed_instances',\n", - " 'component_question_texts', 'is_train', 'type', 'graph', 'graph_size'],\n", - " dtype='object')" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.columns" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df = df[['id', 'question_text', 'answer_text', 'is_train', 'graph', 'type', 'graph_size']]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df.head()\n", - "refined_df.to_csv('wikimultihopqa_refined.csv', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2024-08-07 16:56:03-- https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz?dl=1\n", - "Resolving www.dropbox.com (www.dropbox.com)... 162.125.13.18, 2620:100:6017:18::a27d:212\n", - "Connecting to www.dropbox.com (www.dropbox.com)|162.125.13.18|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://www.dropbox.com/scl/fi/jfqufhpsryt5sdfzaa63o/wikidata5m_alias.tar.gz?rlkey=hsy3o09yxmi5gpqpc2s4myzvh&dl=1 [following]\n", - "--2024-08-07 16:56:03-- https://www.dropbox.com/scl/fi/jfqufhpsryt5sdfzaa63o/wikidata5m_alias.tar.gz?rlkey=hsy3o09yxmi5gpqpc2s4myzvh&dl=1\n", - "Reusing existing connection to www.dropbox.com:443.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline/CYMQuDkY4mm8tz8BDJERS9ji8JJGNUFMRGZuSv3eQrSPj8ggW9-B__HKsfZoqM94GvbZoacWqN_QxovAJZ7c-EUuHhKPCD352hG7vSIYFBEI-5nwo4dmYbE5Ow1_XjRtsNLPJvygFKV_u-wpjzu6naI1/file?dl=1# [following]\n", - "--2024-08-07 16:56:04-- https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline/CYMQuDkY4mm8tz8BDJERS9ji8JJGNUFMRGZuSv3eQrSPj8ggW9-B__HKsfZoqM94GvbZoacWqN_QxovAJZ7c-EUuHhKPCD352hG7vSIYFBEI-5nwo4dmYbE5Ow1_XjRtsNLPJvygFKV_u-wpjzu6naI1/file?dl=1\n", - "Resolving uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com (uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com)... 162.125.13.15, 2620:100:601b:15::a27d:80f\n", - "Connecting to uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com (uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com)|162.125.13.15|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: /cd/0/inline2/CYMRQvw0HUqAAx3Y2uknr49kpFW0HvB1Qi9bPjKxqYAKIeFAkJ8x4G93uodurWf4UqOF3gW-_l3_evOuaRc5p3AYOCVzm3LhHUKPbMX4OW0ZvVa31yiNG0OGGBuZrgn8zVhc3aVpdhlcHUk6IGMK9CFY8uZL3-ZJfV-ckuXh_aAOuaT8aNXuHMCBWoTbiK2YAcJnWIog9ySrO3jxfLlV6QDHoR8gcJH91VePzZgDVkfG9ekXjO1Prbjkzyv8FPPpDyZtsPT9ysq8hbOmpGhwTCXQoApY7QhNG7bHtww4d9beY5O-e5UnaznYJqMbW6Dmc5Jz3zePlXVjAIG-DcnBvyfsQXeCGOO8ZD6f8WyxH3hKLwU0_sSQJqnUx9Kgu4Z5sng/file?dl=1 [following]\n", - "--2024-08-07 16:56:05-- https://uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com/cd/0/inline2/CYMRQvw0HUqAAx3Y2uknr49kpFW0HvB1Qi9bPjKxqYAKIeFAkJ8x4G93uodurWf4UqOF3gW-_l3_evOuaRc5p3AYOCVzm3LhHUKPbMX4OW0ZvVa31yiNG0OGGBuZrgn8zVhc3aVpdhlcHUk6IGMK9CFY8uZL3-ZJfV-ckuXh_aAOuaT8aNXuHMCBWoTbiK2YAcJnWIog9ySrO3jxfLlV6QDHoR8gcJH91VePzZgDVkfG9ekXjO1Prbjkzyv8FPPpDyZtsPT9ysq8hbOmpGhwTCXQoApY7QhNG7bHtww4d9beY5O-e5UnaznYJqMbW6Dmc5Jz3zePlXVjAIG-DcnBvyfsQXeCGOO8ZD6f8WyxH3hKLwU0_sSQJqnUx9Kgu4Z5sng/file?dl=1\n", - "Reusing existing connection to uc2065c05af264864b246a02c2ce.dl.dropboxusercontent.com:443.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 197450113 (188M) [application/binary]\n", - "Saving to: ‘wikidata5m_alias.tar.gz?dl=1’\n", - "\n", - "wikidata5m_alias.ta 100%[===================>] 188.30M 72.7MB/s in 2.6s \n", - "\n", - "2024-08-07 16:56:08 (72.7 MB/s) - ‘wikidata5m_alias.tar.gz?dl=1’ saved [197450113/197450113]\n", - "\n" - ] - } - ], - "source": [ - "!wget \"https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz?dl=1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "wikidata5m_entity.txt\n", - "wikidata5m_relation.txt\n" - ] - } - ], - "source": [ - "!tar -xvf \"wikidata5m_alias.tar.gz?dl=1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "P489\tcurrency symbol description\n", - "\n" - ] - } - ], - "source": [ - "with open('wikidata5m_relation.txt') as f:\n", - " print(f.readline())" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "825it [00:00, 193778.40it/s]\n" - ] - } - ], - "source": [ - "relation_map = {}\n", - "with open('wikidata5m_relation.txt') as f:\n", - " for line in tqdm.tqdm(f):\n", - " parts = line.strip().split('\\t')\n", - " for i in range(1, len(parts)):\n", - " if parts[i] not in relation_map:\n", - " relation_map[parts[i]] = []\n", - " relation_map[parts[i]].append(parts[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "inception\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "inception\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "inception\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "inception\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "editor ['P98', 'P1040']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "date of death\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "inception\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "date of birth\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "date of death\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "date of birth\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "country ['P3005', 'P17']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n", - "director ['P1037', 'P57']\n" - ] - } - ], - "source": [ - "# Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M\n", - "for row in refined_df.itertuples():\n", - " for trip in row.graph:\n", - " relation = trip[1]\n", - " if relation not in relation_map:\n", - " print(relation)\n", - " elif len(relation_map[relation]) > 1:\n", - " print(relation, relation_map[relation])" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "4813491it [00:04, 1109451.61it/s]\n" - ] - } - ], - "source": [ - "entity_set = set()\n", - "with open('wikidata5m_entity.txt') as f:\n", - " for line in tqdm.tqdm(f):\n", - " entity_set.add(line.strip().split('\\t')[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Q7069334\n", - "date_information\n", - "date_information\n", - "Q64768110\n", - "date_information\n", - "date_information\n", - "Q30324147\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63802341\n", - "date_information\n", - "date_information\n", - "Q3983360\n", - "date_information\n", - "date_information\n", - "Q60738284\n", - "date_information\n", - "date_information\n", - "Q12978262\n", - "date_information\n", - "date_information\n", - "Q3372419\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q59962784\n", - "date_information\n", - "date_information\n", - "Q3998173\n", - "date_information\n", - "Q28713010\n", - "Q331352\n", - "Q331352\n", - "Q54829004\n", - "date_information\n", - "Q12830714\n", - "date_information\n", - "Q60738580\n", - "date_information\n", - "Q19796493\n", - "Q39072549\n", - "Q63927168\n", - "date_information\n", - "date_information\n", - "Q1069403\n", - "Q51803959\n", - "date_information\n", - "date_information\n", - "Q65939060\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64174957\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18634612\n", - "date_information\n", - "Q4298967\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48672430\n", - "date_information\n", - "Q19694514\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48674696\n", - "Q48671501\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24483147\n", - "date_information\n", - "Q25106438\n", - "Q55081821\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65086758\n", - "Q2972533\n", - "Q24484294\n", - "Q3989858\n", - "date_information\n", - "date_information\n", - "Q56452642\n", - "date_information\n", - "Q3878817\n", - "date_information\n", - "date_information\n", - "Q64768127\n", - "Q54858959\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4428162\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4289253\n", - "date_information\n", - "Q17084224\n", - "date_information\n", - "Q1350154\n", - "date_information\n", - "date_information\n", - "Q64138712\n", - "Q76401690\n", - "Q3707073\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q3680696\n", - "Q3680696\n", - "date_information\n", - "date_information\n", - "Q63248839\n", - "date_information\n", - "Q54868320\n", - "date_information\n", - "date_information\n", - "Q60745215\n", - "Q50314058\n", - "Q3627346\n", - "date_information\n", - "date_information\n", - "Q3374809\n", - "Q3374809\n", - "Q64768701\n", - "Q992815\n", - "Q992815\n", - "Q15622114\n", - "Q28064471\n", - "Q60737563\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60456550\n", - "Q52151279\n", - "date_information\n", - "date_information\n", - "Q69827389\n", - "date_information\n", - "date_information\n", - "Q61958912\n", - "Q27\n", - "Q67130286\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58313464\n", - "date_information\n", - "Q56550941\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56284087\n", - "date_information\n", - "Q48672392\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60750745\n", - "Q18340884\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55403801\n", - "Q55403801\n", - "Q55606456\n", - "Q48673905\n", - "Q60737958\n", - "Q6214794\n", - "Q6214794\n", - "date_information\n", - "date_information\n", - "Q65924962\n", - "date_information\n", - "date_information\n", - "Q13788301\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q55595776\n", - "date_information\n", - "date_information\n", - "Q62129437\n", - "date_information\n", - "Q60738487\n", - "date_information\n", - "Q50819333\n", - "Q43388529\n", - "Q43388529\n", - "date_information\n", - "date_information\n", - "Q23814892\n", - "date_information\n", - "date_information\n", - "Q16309250\n", - "Q30898860\n", - "Q56241165\n", - "date_information\n", - "Q55419710\n", - "Q55419710\n", - "date_information\n", - "date_information\n", - "Q47090972\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19760848\n", - "Q48426793\n", - "Q34956353\n", - "date_information\n", - "Q63183587\n", - "date_information\n", - "Q61456874\n", - "Q45900448\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30623557\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3002150\n", - "Q23814892\n", - "Q4109853\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q59617591\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64577356\n", - "date_information\n", - "Q55102619\n", - "date_information\n", - "Q3742313\n", - "date_information\n", - "Q51885931\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64759209\n", - "Q2365600\n", - "Q57246562\n", - "Q15681455\n", - "date_information\n", - "date_information\n", - "Q64768836\n", - "date_information\n", - "date_information\n", - "Q6506487\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1068991\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65032686\n", - "date_information\n", - "date_information\n", - "Q60748551\n", - "date_information\n", - "Q56276953\n", - "date_information\n", - "Q11081421\n", - "date_information\n", - "Q21336011\n", - "Q56279611\n", - "date_information\n", - "Q12290762\n", - "Q3680696\n", - "Q3680696\n", - "date_information\n", - "date_information\n", - "Q53558909\n", - "Q2143090\n", - "Q6033589\n", - "Q6033589\n", - "Q56426852\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4844692\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q60549164\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3425284\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19329824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17079176\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48671589\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56152806\n", - "date_information\n", - "Q60738551\n", - "date_information\n", - "date_information\n", - "Q64577409\n", - "date_information\n", - "Q63969266\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q23976970\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q59728996\n", - "Q56279069\n", - "Q54868294\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47004710\n", - "date_information\n", - "Q64140895\n", - "date_information\n", - "date_information\n", - "Q10304581\n", - "Q10304581\n", - "date_information\n", - "Q20644750\n", - "Q76890341\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q76014955\n", - "Q2061015\n", - "Q64767767\n", - "date_information\n", - "Q51613741\n", - "Q27536286\n", - "Q56045065\n", - "date_information\n", - "Q63698633\n", - "Q17372365\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19884631\n", - "date_information\n", - "Q64876881\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65677528\n", - "Q17237979\n", - "Q17237979\n", - "date_information\n", - "date_information\n", - "Q17086825\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56063323\n", - "Q63802332\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63686494\n", - "Q55991302\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q5544317\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q57983083\n", - "Q55614153\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64355175\n", - "Q16310763\n", - "date_information\n", - "Q24189885\n", - "date_information\n", - "Q54020505\n", - "date_information\n", - "Q26921021\n", - "Q26921021\n", - "date_information\n", - "Q21168625\n", - "date_information\n", - "Q65086649\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12976077\n", - "Q9005\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q17124784\n", - "Q46992828\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20076298\n", - "Q1069296\n", - "Q47487574\n", - "date_information\n", - "Q4318541\n", - "Q51077490\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56692191\n", - "Q22043722\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3978256\n", - "Q3978256\n", - "date_information\n", - "date_information\n", - "Q19884760\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60315862\n", - "date_information\n", - "Q60738035\n", - "date_information\n", - "Q48674000\n", - "Q16568988\n", - "date_information\n", - "date_information\n", - "Q75279216\n", - "date_information\n", - "Q76488781\n", - "Q4063775\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q13629051\n", - "Q7014940\n", - "Q7014940\n", - "date_information\n", - "Q4449340\n", - "Q55607254\n", - "date_information\n", - "Q6060947\n", - "date_information\n", - "date_information\n", - "Q3825078\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3012352\n", - "Q18643972\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56086939\n", - "Q18160958\n", - "Q62020324\n", - "Q57314409\n", - "Q66764374\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47467250\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21639250\n", - "date_information\n", - "Q17084381\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65091853\n", - "Q14333308\n", - "date_information\n", - "Q1057286\n", - "date_information\n", - "Q25104948\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63969278\n", - "Q31734750\n", - "date_information\n", - "date_information\n", - "Q48772026\n", - "date_information\n", - "Q3680696\n", - "Q3680696\n", - "Q50650139\n", - "Q45040758\n", - "Q48673974\n", - "date_information\n", - "date_information\n", - "Q56042875\n", - "Q19884834\n", - "date_information\n", - "date_information\n", - "Q55806689\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q456383\n", - "Q456383\n", - "date_information\n", - "date_information\n", - "Q4184537\n", - "date_information\n", - "Q55768557\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2781903\n", - "Q2781903\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65032573\n", - "date_information\n", - "date_information\n", - "Q4459376\n", - "Q55621958\n", - "date_information\n", - "date_information\n", - "Q2157653\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q5706176\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56703335\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q331352\n", - "Q331352\n", - "Q63802265\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51955042\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24261792\n", - "Q59393445\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62023950\n", - "date_information\n", - "Q72422839\n", - "date_information\n", - "date_information\n", - "Q65032648\n", - "Q66305376\n", - "Q55227056\n", - "Q55227056\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q43398827\n", - "Q2570066\n", - "Q4131248\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q10655046\n", - "date_information\n", - "Q60790967\n", - "date_information\n", - "date_information\n", - "Q18554460\n", - "Q48671922\n", - "Q48757939\n", - "Q1062145\n", - "Q25505760\n", - "Q4365301\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q5253994\n", - "Q27530932\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25505760\n", - "Q28457976\n", - "date_information\n", - "date_information\n", - "Q46230954\n", - "Q46230954\n", - "Q16625133\n", - "date_information\n", - "Q64759208\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30312756\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q71050994\n", - "Q59167929\n", - "Q47500739\n", - "date_information\n", - "date_information\n", - "Q75274558\n", - "date_information\n", - "Q19690966\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q72280034\n", - "date_information\n", - "date_information\n", - "Q63994392\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55641238\n", - "Q56063528\n", - "date_information\n", - "date_information\n", - "Q11983511\n", - "date_information\n", - "date_information\n", - "Q65677545\n", - "date_information\n", - "Q17478875\n", - "date_information\n", - "date_information\n", - "Q48798236\n", - "Q60737736\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58214420\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63924529\n", - "Q62595482\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q48674262\n", - "date_information\n", - "Q57584335\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65052620\n", - "date_information\n", - "Q60737570\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47488371\n", - "Q76014955\n", - "Q28434505\n", - "Q66305376\n", - "date_information\n", - "date_information\n", - "Q55317987\n", - "date_information\n", - "Q55363997\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64211107\n", - "date_information\n", - "date_information\n", - "Q61450772\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15143923\n", - "Q2159814\n", - "Q48671626\n", - "Q56284087\n", - "Q51540879\n", - "date_information\n", - "date_information\n", - "Q23900134\n", - "date_information\n", - "Q75276877\n", - "Q64768050\n", - "Q61365850\n", - "Q61365850\n", - "date_information\n", - "Q1049634\n", - "date_information\n", - "Q48671205\n", - "date_information\n", - "date_information\n", - "Q64018111\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62003783\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q10388675\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q49700441\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60062917\n", - "date_information\n", - "Q57328748\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4171785\n", - "Q60737762\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19374242\n", - "date_information\n", - "Q60637519\n", - "date_information\n", - "date_information\n", - "Q5438207\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55333388\n", - "date_information\n", - "Q55165507\n", - "Q55081823\n", - "date_information\n", - "date_information\n", - "Q50295992\n", - "Q3808084\n", - "date_information\n", - "Q22075101\n", - "date_information\n", - "date_information\n", - "Q12232417\n", - "Q30633631\n", - "Q63924529\n", - "Q2907144\n", - "date_information\n", - "date_information\n", - "Q11161132\n", - "Q55772702\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62277831\n", - "Q71246482\n", - "date_information\n", - "date_information\n", - "Q56615611\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21336011\n", - "Q62018327\n", - "date_information\n", - "Q27\n", - "Q45763740\n", - "date_information\n", - "Q55612903\n", - "Q60767894\n", - "Q19887333\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1087285\n", - "Q3321378\n", - "Q64768647\n", - "Q2282608\n", - "date_information\n", - "date_information\n", - "Q78305911\n", - "date_information\n", - "Q3304810\n", - "Q15270573\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63698573\n", - "date_information\n", - "date_information\n", - "Q60745185\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q45046451\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3567855\n", - "Q7936508\n", - "Q7936508\n", - "date_information\n", - "date_information\n", - "Q60738633\n", - "Q64627115\n", - "date_information\n", - "Q16614474\n", - "date_information\n", - "Q56322655\n", - "Q52715786\n", - "date_information\n", - "Q3419202\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65678415\n", - "Q66124718\n", - "Q66124718\n", - "date_information\n", - "date_information\n", - "Q62860838\n", - "Q62860838\n", - "Q63986511\n", - "Q63986512\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15706063\n", - "Q3958334\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12988947\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24659886\n", - "Q75521509\n", - "Q26935570\n", - "Q48790611\n", - "Q4378422\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48674657\n", - "Q55647759\n", - "date_information\n", - "Q65080213\n", - "Q66688160\n", - "Q58814919\n", - "Q76273253\n", - "Q19908291\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58814888\n", - "date_information\n", - "Q60738293\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56401290\n", - "date_information\n", - "Q3549581\n", - "Q56596744\n", - "date_information\n", - "date_information\n", - "Q10406203\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19618477\n", - "date_information\n", - "date_information\n", - "Q13009424\n", - "date_information\n", - "Q75249592\n", - "date_information\n", - "Q56045974\n", - "Q48835993\n", - "Q23073331\n", - "Q55767505\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q19952744\n", - "Q63450254\n", - "date_information\n", - "date_information\n", - "Q16513344\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q59728996\n", - "Q51955042\n", - "date_information\n", - "Q52158771\n", - "Q63398302\n", - "Q55450652\n", - "Q27\n", - "Q44485312\n", - "Q60737553\n", - "Q75804226\n", - "date_information\n", - "date_information\n", - "Q56416462\n", - "date_information\n", - "Q75253659\n", - "date_information\n", - "date_information\n", - "Q60197034\n", - "date_information\n", - "date_information\n", - "Q15905014\n", - "date_information\n", - "date_information\n", - "Q63969256\n", - "date_information\n", - "Q10322276\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18405892\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50650382\n", - "date_information\n", - "Q76014955\n", - "date_information\n", - "Q3874453\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30621521\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60737691\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66828708\n", - "date_information\n", - "Q75497811\n", - "Q63565874\n", - "date_information\n", - "date_information\n", - "Q11048250\n", - "date_information\n", - "Q1061054\n", - "date_information\n", - "date_information\n", - "Q4130286\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q76186239\n", - "Q18518927\n", - "Q18518927\n", - "date_information\n", - "Q28406793\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q27\n", - "Q58815150\n", - "date_information\n", - "Q18364314\n", - "date_information\n", - "Q60734621\n", - "date_information\n", - "date_information\n", - "Q75492515\n", - "Q64549196\n", - "date_information\n", - "Q63119071\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16028461\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19655285\n", - "date_information\n", - "date_information\n", - "Q19618531\n", - "Q56816889\n", - "date_information\n", - "Q62075985\n", - "date_information\n", - "Q73538385\n", - "date_information\n", - "date_information\n", - "Q12976368\n", - "Q29912823\n", - "date_information\n", - "date_information\n", - "Q15689850\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24940239\n", - "date_information\n", - "Q24729251\n", - "Q63246549\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64485310\n", - "Q15623497\n", - "Q20005025\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56416519\n", - "date_information\n", - "Q59917796\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77885110\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q46600570\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58411267\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47004710\n", - "Q1388181\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75574603\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63924479\n", - "Q64460079\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55726186\n", - "date_information\n", - "date_information\n", - "Q30880265\n", - "date_information\n", - "date_information\n", - "Q54020341\n", - "Q48672538\n", - "date_information\n", - "date_information\n", - "Q48674649\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55590066\n", - "date_information\n", - "Q66363590\n", - "Q3522860\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56703332\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20093154\n", - "Q65082453\n", - "date_information\n", - "Q65320559\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55637523\n", - "Q47500749\n", - "date_information\n", - "Q60165041\n", - "Q4076139\n", - "date_information\n", - "Q61450811\n", - "Q65678838\n", - "date_information\n", - "Q24728788\n", - "Q63183577\n", - "Q6966375\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47004852\n", - "date_information\n", - "Q11982970\n", - "Q77892959\n", - "date_information\n", - "Q21337743\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25610585\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q8557599\n", - "date_information\n", - "Q47455556\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60783304\n", - "date_information\n", - "Q12977047\n", - "Q66588063\n", - "date_information\n", - "Q19884740\n", - "Q2929293\n", - "date_information\n", - "Q60463503\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4108587\n", - "date_information\n", - "Q24672337\n", - "Q24672337\n", - "Q12943321\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50728314\n", - "Q4078557\n", - "Q58754776\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q77892476\n", - "date_information\n", - "date_information\n", - "Q53674897\n", - "Q3018197\n", - "Q3018197\n", - "date_information\n", - "Q55393988\n", - "date_information\n", - "Q13640080\n", - "date_information\n", - "Q28457675\n", - "date_information\n", - "date_information\n", - "Q12008229\n", - "Q60764980\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64460234\n", - "date_information\n", - "Q5675398\n", - "Q48674553\n", - "Q27703697\n", - "Q64768590\n", - "date_information\n", - "Q48671619\n", - "date_information\n", - "date_information\n", - "Q64768319\n", - "date_information\n", - "date_information\n", - "Q66840990\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q64768648\n", - "Q16016727\n", - "Q74193018\n", - "date_information\n", - "Q57053277\n", - "Q57053277\n", - "date_information\n", - "date_information\n", - "Q331352\n", - "Q331352\n", - "Q62073359\n", - "Q56279425\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3754764\n", - "date_information\n", - "date_information\n", - "Q21852592\n", - "date_information\n", - "Q56063218\n", - "date_information\n", - "date_information\n", - "Q1913674\n", - "date_information\n", - "Q1475421\n", - "date_information\n", - "Q50755863\n", - "Q28457524\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52151200\n", - "Q65064897\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q10791093\n", - "Q55258973\n", - "Q3289992\n", - "date_information\n", - "Q997387\n", - "Q997387\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1056743\n", - "Q27\n", - "Q64830343\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q29912823\n", - "date_information\n", - "Q16134448\n", - "Q16134448\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60737621\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q41787227\n", - "Q56279002\n", - "date_information\n", - "date_information\n", - "Q17199295\n", - "date_information\n", - "date_information\n", - "Q22978862\n", - "date_information\n", - "Q2625914\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3213381\n", - "date_information\n", - "Q55621609\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q10952924\n", - "date_information\n", - "Q79120099\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q39075975\n", - "date_information\n", - "Q50450145\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q16440157\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1054847\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12930727\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19278235\n", - "Q60737632\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21843347\n", - "date_information\n", - "date_information\n", - "Q19521612\n", - "date_information\n", - "date_information\n", - "Q58437727\n", - "date_information\n", - "date_information\n", - "Q56537271\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2940176\n", - "date_information\n", - "Q55611501\n", - "Q60737676\n", - "Q60738269\n", - "date_information\n", - "Q4108743\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56289818\n", - "date_information\n", - "Q21343277\n", - "date_information\n", - "Q2900415\n", - "date_information\n", - "Q64755607\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28147010\n", - "date_information\n", - "date_information\n", - "Q19387802\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16028548\n", - "Q77892399\n", - "Q44821118\n", - "Q59305418\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4340096\n", - "date_information\n", - "date_information\n", - "Q4207072\n", - "Q4296236\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65061089\n", - "date_information\n", - "Q47452391\n", - "Q19907507\n", - "Q55907550\n", - "Q52145034\n", - "Q66764383\n", - "Q60738164\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16253118\n", - "date_information\n", - "Q16015511\n", - "Q60646525\n", - "date_information\n", - "Q54868306\n", - "date_information\n", - "date_information\n", - "Q56452637\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66841330\n", - "Q54020757\n", - "date_information\n", - "Q47450881\n", - "date_information\n", - "Q47081778\n", - "date_information\n", - "Q60439387\n", - "date_information\n", - "Q57584371\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52276380\n", - "Q65087828\n", - "date_information\n", - "date_information\n", - "Q17416360\n", - "date_information\n", - "Q12977309\n", - "date_information\n", - "Q38049163\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50074317\n", - "date_information\n", - "Q50414212\n", - "Q27\n", - "Q2302896\n", - "Q2302896\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3429285\n", - "Q4237108\n", - "Q56241172\n", - "date_information\n", - "Q55552345\n", - "date_information\n", - "Q56703329\n", - "Q52715793\n", - "date_information\n", - "Q40745153\n", - "date_information\n", - "date_information\n", - "Q592788\n", - "date_information\n", - "Q3192937\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48996920\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21040363\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30594579\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64174444\n", - "date_information\n", - "Q54879127\n", - "date_information\n", - "date_information\n", - "Q23655553\n", - "Q23655553\n", - "date_information\n", - "date_information\n", - "Q55611897\n", - "date_information\n", - "date_information\n", - "Q31310887\n", - "Q65045015\n", - "Q3221009\n", - "Q52776612\n", - "Q64768124\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28220176\n", - "date_information\n", - "date_information\n", - "Q2590303\n", - "Q12991096\n", - "date_information\n", - "Q1070043\n", - "date_information\n", - "date_information\n", - "Q63119112\n", - "Q51730226\n", - "Q51730226\n", - "date_information\n", - "Q47034913\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25505742\n", - "Q15721083\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63924372\n", - "Q62631136\n", - "Q47307336\n", - "date_information\n", - "date_information\n", - "Q53854471\n", - "date_information\n", - "date_information\n", - "Q30633641\n", - "Q13570759\n", - "Q3749311\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56037055\n", - "Q58843878\n", - "Q48815037\n", - "date_information\n", - "Q1062023\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12439936\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18086419\n", - "Q56452620\n", - "date_information\n", - "Q56692195\n", - "date_information\n", - "Q16316783\n", - "date_information\n", - "Q63020397\n", - "Q45192617\n", - "Q45192617\n", - "date_information\n", - "Q8851702\n", - "date_information\n", - "Q55612948\n", - "date_information\n", - "Q65081567\n", - "Q56416519\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56275844\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2326685\n", - "Q18554460\n", - "date_information\n", - "date_information\n", - "Q47513340\n", - "date_information\n", - "Q48784694\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51077300\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21183743\n", - "date_information\n", - "Q19329824\n", - "Q27\n", - "Q49239599\n", - "date_information\n", - "Q50825750\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q53374241\n", - "date_information\n", - "date_information\n", - "Q16633939\n", - "date_information\n", - "Q47064237\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738635\n", - "Q64768137\n", - "Q69629123\n", - "date_information\n", - "Q77892368\n", - "date_information\n", - "Q47455814\n", - "Q63247492\n", - "Q60738560\n", - "date_information\n", - "Q55633963\n", - "Q21081425\n", - "Q50825136\n", - "Q50825136\n", - "date_information\n", - "date_information\n", - "Q27535196\n", - "Q65073215\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4102998\n", - "date_information\n", - "Q56281825\n", - "date_information\n", - "date_information\n", - "Q65924942\n", - "Q67146474\n", - "Q47034953\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24261792\n", - "date_information\n", - "date_information\n", - "Q3858110\n", - "date_information\n", - "Q55450595\n", - "Q56600254\n", - "date_information\n", - "date_information\n", - "Q66691283\n", - "date_information\n", - "date_information\n", - "Q74190856\n", - "date_information\n", - "Q3025000\n", - "date_information\n", - "date_information\n", - "Q55637599\n", - "Q3521000\n", - "date_information\n", - "Q64211103\n", - "Q30738154\n", - "Q48671816\n", - "date_information\n", - "Q25619745\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q43384713\n", - "date_information\n", - "date_information\n", - "Q56279529\n", - "Q47063859\n", - "Q47455556\n", - "date_information\n", - "Q17082113\n", - "date_information\n", - "date_information\n", - "Q17079363\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54229562\n", - "Q19907720\n", - "date_information\n", - "date_information\n", - "Q47038485\n", - "date_information\n", - "Q43303325\n", - "date_information\n", - "date_information\n", - "Q55623763\n", - "Q55623763\n", - "date_information\n", - "Q57780001\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q67222754\n", - "Q65071895\n", - "date_information\n", - "Q55696425\n", - "date_information\n", - "Q11634137\n", - "date_information\n", - "Q3986259\n", - "Q55615148\n", - "Q55623864\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56651740\n", - "Q62278093\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q63698666\n", - "date_information\n", - "date_information\n", - "Q46606647\n", - "Q56249739\n", - "Q64338322\n", - "Q56249739\n", - "Q66459265\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1192808\n", - "Q4246432\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64577360\n", - "Q56405840\n", - "Q15633656\n", - "date_information\n", - "date_information\n", - "Q5668844\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16679535\n", - "date_information\n", - "Q7377305\n", - "Q4199413\n", - "Q63735869\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63001772\n", - "date_information\n", - "Q60191520\n", - "Q15926680\n", - "date_information\n", - "Q64140888\n", - "date_information\n", - "Q6161423\n", - "date_information\n", - "date_information\n", - "Q25738584\n", - "date_information\n", - "Q48672646\n", - "Q54021017\n", - "Q62631150\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q55625195\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18518927\n", - "date_information\n", - "date_information\n", - "Q56276591\n", - "Q73536300\n", - "date_information\n", - "Q61972627\n", - "date_information\n", - "Q28690154\n", - "date_information\n", - "Q60310526\n", - "date_information\n", - "Q48816553\n", - "date_information\n", - "date_information\n", - "Q25539787\n", - "Q31295811\n", - "date_information\n", - "Q24484294\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60683565\n", - "Q60690251\n", - "Q60690251\n", - "Q21672570\n", - "Q65679595\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64759228\n", - "Q48674397\n", - "date_information\n", - "Q48673900\n", - "date_information\n", - "Q4508339\n", - "Q52151214\n", - "Q12707541\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56045021\n", - "date_information\n", - "Q3192937\n", - "Q61037667\n", - "date_information\n", - "Q60738529\n", - "date_information\n", - "Q3829140\n", - "Q3849100\n", - "Q3849100\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63565887\n", - "date_information\n", - "date_information\n", - "Q52715614\n", - "date_information\n", - "Q12974353\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738058\n", - "date_information\n", - "date_information\n", - "Q44492063\n", - "Q69629123\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768391\n", - "Q52715740\n", - "Q4210161\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48805436\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55627581\n", - "Q55598788\n", - "Q20978738\n", - "Q20978738\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48739903\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52320451\n", - "date_information\n", - "date_information\n", - "Q38251170\n", - "Q3724386\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63248850\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20245404\n", - "Q48671499\n", - "Q51285350\n", - "Q75742719\n", - "date_information\n", - "date_information\n", - "Q30738154\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12924557\n", - "date_information\n", - "date_information\n", - "Q60738403\n", - "Q60738268\n", - "date_information\n", - "Q77895522\n", - "date_information\n", - "date_information\n", - "Q61669648\n", - "date_information\n", - "Q55632769\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25508485\n", - "Q55611631\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17080137\n", - "date_information\n", - "Q19886924\n", - "date_information\n", - "Q55278849\n", - "Q47542267\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q46600495\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65057861\n", - "Q27\n", - "Q45710417\n", - "date_information\n", - "date_information\n", - "Q63372375\n", - "date_information\n", - "date_information\n", - "Q47011389\n", - "date_information\n", - "Q24728189\n", - "date_information\n", - "Q60745160\n", - "Q55616544\n", - "date_information\n", - "date_information\n", - "Q61747353\n", - "Q28509912\n", - "date_information\n", - "date_information\n", - "Q56604157\n", - "Q61896055\n", - "date_information\n", - "date_information\n", - "Q61747347\n", - "date_information\n", - "date_information\n", - "Q64708093\n", - "Q16664499\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q46369290\n", - "Q60875748\n", - "date_information\n", - "Q4478405\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55609409\n", - "Q64848293\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738412\n", - "date_information\n", - "Q64939837\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55631173\n", - "Q60848339\n", - "Q55635810\n", - "Q55758066\n", - "date_information\n", - "Q19946737\n", - "date_information\n", - "date_information\n", - "Q62595516\n", - "Q23014197\n", - "Q47528044\n", - "Q60787303\n", - "date_information\n", - "Q27963551\n", - "date_information\n", - "Q66364883\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q18713310\n", - "Q55694395\n", - "date_information\n", - "date_information\n", - "Q4319017\n", - "date_information\n", - "date_information\n", - "Q48672721\n", - "Q6503488\n", - "date_information\n", - "Q24153864\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55946142\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54868325\n", - "date_information\n", - "Q48970825\n", - "Q48671199\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50309110\n", - "Q44606399\n", - "date_information\n", - "Q13640271\n", - "date_information\n", - "date_information\n", - "Q59149494\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65087828\n", - "date_information\n", - "Q59692468\n", - "Q56048936\n", - "Q19613178\n", - "date_information\n", - "Q64338245\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48988129\n", - "date_information\n", - "date_information\n", - "Q48671581\n", - "date_information\n", - "date_information\n", - "Q48684414\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4211954\n", - "Q15224621\n", - "date_information\n", - "Q45899823\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55605752\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19614422\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56070897\n", - "date_information\n", - "Q19614201\n", - "Q1052809\n", - "Q22917273\n", - "date_information\n", - "Q60737739\n", - "date_information\n", - "Q1568748\n", - "date_information\n", - "Q56274437\n", - "Q15514334\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63565879\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60748445\n", - "date_information\n", - "Q17478936\n", - "date_information\n", - "date_information\n", - "Q64768489\n", - "Q52063072\n", - "date_information\n", - "date_information\n", - "Q3482558\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q72380423\n", - "Q60734654\n", - "Q26213102\n", - "date_information\n", - "date_information\n", - "Q27536269\n", - "date_information\n", - "date_information\n", - "Q1053778\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61664915\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60565417\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q6172033\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60487164\n", - "date_information\n", - "Q29643416\n", - "date_information\n", - "Q20245957\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24975344\n", - "date_information\n", - "Q47009365\n", - "date_information\n", - "Q65678112\n", - "date_information\n", - "date_information\n", - "Q55635268\n", - "date_information\n", - "Q64768620\n", - "date_information\n", - "date_information\n", - "Q1285133\n", - "date_information\n", - "Q56278697\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q70887035\n", - "Q53717659\n", - "Q15635899\n", - "Q15635899\n", - "Q7881655\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q45125179\n", - "date_information\n", - "Q48671573\n", - "Q56598902\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54829004\n", - "Q10441356\n", - "Q64768153\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60370884\n", - "Q60785708\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q53912009\n", - "Q54819554\n", - "Q63450179\n", - "Q7271631\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q65051886\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47468248\n", - "date_information\n", - "date_information\n", - "Q4009651\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63643994\n", - "Q47500766\n", - "Q4053879\n", - "Q60738710\n", - "date_information\n", - "Q62031912\n", - "date_information\n", - "date_information\n", - "Q62277405\n", - "date_information\n", - "date_information\n", - "Q51999611\n", - "date_information\n", - "Q17639864\n", - "date_information\n", - "Q331352\n", - "Q331352\n", - "date_information\n", - "date_information\n", - "Q65090819\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3522853\n", - "Q4193938\n", - "date_information\n", - "Q49885354\n", - "date_information\n", - "date_information\n", - "Q12985827\n", - "date_information\n", - "Q55258983\n", - "date_information\n", - "Q57314412\n", - "date_information\n", - "Q1631673\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3296267\n", - "date_information\n", - "Q48674522\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56416519\n", - "Q18643972\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q7936508\n", - "Q7936508\n", - "Q56092489\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56615611\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52157923\n", - "date_information\n", - "Q60738619\n", - "Q54021044\n", - "date_information\n", - "date_information\n", - "Q55635199\n", - "date_information\n", - "Q19619138\n", - "Q64768680\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4213798\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4378254\n", - "date_information\n", - "Q62595390\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48804187\n", - "Q48674213\n", - "Q66305379\n", - "Q3429285\n", - "date_information\n", - "Q3670793\n", - "Q4289625\n", - "date_information\n", - "date_information\n", - "Q3986504\n", - "Q67266090\n", - "Q33170323\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12537376\n", - "Q53613358\n", - "date_information\n", - "date_information\n", - "Q55759051\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20245957\n", - "date_information\n", - "date_information\n", - "Q18500792\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1056331\n", - "Q11884841\n", - "Q64768732\n", - "Q65065675\n", - "date_information\n", - "Q31192565\n", - "Q1067104\n", - "Q59124886\n", - "Q59124886\n", - "date_information\n", - "Q30684118\n", - "date_information\n", - "Q51278968\n", - "date_information\n", - "date_information\n", - "Q27527841\n", - "Q4255603\n", - "Q62018324\n", - "date_information\n", - "date_information\n", - "Q56284302\n", - "date_information\n", - "date_information\n", - "Q77896663\n", - "Q56452587\n", - "Q3314521\n", - "Q3314521\n", - "date_information\n", - "Q27\n", - "Q4734672\n", - "date_information\n", - "Q12991152\n", - "date_information\n", - "date_information\n", - "Q65924938\n", - "date_information\n", - "Q60737546\n", - "date_information\n", - "Q48318033\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q6418615\n", - "Q6418615\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q13564944\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4199990\n", - "date_information\n", - "date_information\n", - "Q17478107\n", - "date_information\n", - "date_information\n", - "Q48672665\n", - "date_information\n", - "Q16557779\n", - "Q1068486\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2163100\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q50649657\n", - "date_information\n", - "date_information\n", - "Q56045047\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q8837872\n", - "date_information\n", - "Q59617591\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63398838\n", - "date_information\n", - "Q65679587\n", - "date_information\n", - "date_information\n", - "Q50650382\n", - "Q65552652\n", - "Q66455732\n", - "date_information\n", - "date_information\n", - "Q67147057\n", - "Q56291009\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4742176\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63869121\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1024824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3422649\n", - "Q21342153\n", - "date_information\n", - "Q15224531\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56816874\n", - "date_information\n", - "Q47455556\n", - "date_information\n", - "date_information\n", - "Q65065129\n", - "Q25490712\n", - "Q54020295\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15702124\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58814916\n", - "Q50649795\n", - "date_information\n", - "Q55583728\n", - "Q4423129\n", - "date_information\n", - "date_information\n", - "Q2571155\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q63565895\n", - "date_information\n", - "Q62018336\n", - "date_information\n", - "date_information\n", - "Q75278776\n", - "date_information\n", - "Q11339934\n", - "Q19618217\n", - "date_information\n", - "Q20004254\n", - "Q56071282\n", - "Q48818141\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61750746\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18554460\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q11824526\n", - "Q30901566\n", - "date_information\n", - "Q19896174\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56248006\n", - "Q56248006\n", - "date_information\n", - "date_information\n", - "Q5659177\n", - "Q16719675\n", - "date_information\n", - "Q63923824\n", - "date_information\n", - "Q49750686\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25396637\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19848388\n", - "Q331352\n", - "Q331352\n", - "Q45302060\n", - "Q11993882\n", - "Q53854471\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56756930\n", - "date_information\n", - "Q55762020\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15615493\n", - "date_information\n", - "date_information\n", - "Q4313070\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51278938\n", - "Q60738710\n", - "date_information\n", - "Q48835993\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738182\n", - "Q64577408\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60740385\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1583694\n", - "Q16766207\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48673905\n", - "Q56886895\n", - "date_information\n", - "Q55625517\n", - "date_information\n", - "Q66498260\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4298967\n", - "Q64597510\n", - "date_information\n", - "Q29168841\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48422401\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77906377\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q39058429\n", - "Q46840692\n", - "Q47091822\n", - "Q47091822\n", - "date_information\n", - "date_information\n", - "Q54020520\n", - "Q48671517\n", - "Q3749740\n", - "Q56678187\n", - "date_information\n", - "Q2819749\n", - "date_information\n", - "Q55633312\n", - "Q55605990\n", - "Q56249739\n", - "Q56249739\n", - "date_information\n", - "Q17478906\n", - "date_information\n", - "Q37103689\n", - "Q64768157\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q31899662\n", - "date_information\n", - "Q25505901\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16614618\n", - "date_information\n", - "Q56291196\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q45710167\n", - "Q55629777\n", - "Q21672950\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q63248839\n", - "date_information\n", - "Q47455556\n", - "date_information\n", - "Q17154585\n", - "date_information\n", - "Q11920998\n", - "Q55607176\n", - "date_information\n", - "Q68681420\n", - "date_information\n", - "Q4189600\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q44506078\n", - "Q27\n", - "date_information\n", - "Q60737759\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q8826702\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65072188\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15805124\n", - "Q77892368\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q46606577\n", - "date_information\n", - "Q16028548\n", - "Q56274942\n", - "date_information\n", - "date_information\n", - "Q51031293\n", - "date_information\n", - "Q2583655\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q57773022\n", - "Q63969266\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56458840\n", - "date_information\n", - "date_information\n", - "Q60738177\n", - "Q59733171\n", - "date_information\n", - "Q56614298\n", - "Q49114543\n", - "Q56886875\n", - "date_information\n", - "date_information\n", - "Q22000108\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65045109\n", - "date_information\n", - "Q56452546\n", - "Q56334062\n", - "Q3824783\n", - "Q56334062\n", - "Q60738269\n", - "Q3563923\n", - "Q18471775\n", - "Q3563923\n", - "Q4278661\n", - "Q51031341\n", - "Q5990904\n", - "Q51756485\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56537271\n", - "date_information\n", - "date_information\n", - "Q55635194\n", - "date_information\n", - "Q47500692\n", - "date_information\n", - "Q62277379\n", - "date_information\n", - "Q58814919\n", - "date_information\n", - "Q3798546\n", - "Q55098902\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56045065\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56063974\n", - "date_information\n", - "date_information\n", - "Q77903976\n", - "Q64768676\n", - "date_information\n", - "date_information\n", - "Q2134675\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18554460\n", - "Q56874855\n", - "Q51879925\n", - "date_information\n", - "Q66828800\n", - "Q12991096\n", - "Q19848394\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62278019\n", - "Q19329824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64691762\n", - "date_information\n", - "Q52151197\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56615750\n", - "Q12930727\n", - "Q64338322\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64338187\n", - "date_information\n", - "date_information\n", - "Q18406892\n", - "Q19916458\n", - "date_information\n", - "Q57983086\n", - "date_information\n", - "Q43401872\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3990127\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51834363\n", - "date_information\n", - "Q48671238\n", - "Q55637590\n", - "Q64221113\n", - "date_information\n", - "Q48817737\n", - "date_information\n", - "Q53036569\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48671709\n", - "date_information\n", - "Q64768132\n", - "Q65058947\n", - "date_information\n", - "date_information\n", - "Q28466514\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q67628998\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65087118\n", - "Q12002018\n", - "date_information\n", - "Q48671879\n", - "date_information\n", - "Q59836204\n", - "date_information\n", - "date_information\n", - "Q9094439\n", - "date_information\n", - "date_information\n", - "Q2657975\n", - "Q3986523\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q60737958\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56886874\n", - "Q56703333\n", - "Q27\n", - "Q56276944\n", - "Q64768061\n", - "Q63450091\n", - "Q48672731\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19907410\n", - "date_information\n", - "Q3208390\n", - "date_information\n", - "Q55609415\n", - "Q64577347\n", - "Q48990162\n", - "Q48990162\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19613400\n", - "Q24175518\n", - "Q77895479\n", - "date_information\n", - "Q12205404\n", - "Q60738280\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48898535\n", - "date_information\n", - "Q60727937\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q79212167\n", - "date_information\n", - "Q56279327\n", - "Q64211126\n", - "Q44727126\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24728755\n", - "date_information\n", - "date_information\n", - "Q25738039\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65091022\n", - "date_information\n", - "Q65491495\n", - "date_information\n", - "date_information\n", - "Q59728996\n", - "Q16592335\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58094575\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q6560644\n", - "date_information\n", - "date_information\n", - "Q47038485\n", - "date_information\n", - "Q58310549\n", - "date_information\n", - "Q21335696\n", - "Q3192937\n", - "date_information\n", - "date_information\n", - "Q1062924\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q47455556\n", - "Q44417422\n", - "Q61510092\n", - "date_information\n", - "date_information\n", - "Q48672572\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3010170\n", - "date_information\n", - "date_information\n", - "Q56275660\n", - "date_information\n", - "Q24484703\n", - "Q66764359\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52151277\n", - "Q46152071\n", - "date_information\n", - "date_information\n", - "Q25447892\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q11824571\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1050472\n", - "date_information\n", - "Q31189596\n", - "date_information\n", - "Q4297640\n", - "Q48770599\n", - "Q16679476\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30738154\n", - "Q16315884\n", - "date_information\n", - "Q62761097\n", - "date_information\n", - "Q48965733\n", - "date_information\n", - "Q55390546\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51756485\n", - "date_information\n", - "Q55621373\n", - "date_information\n", - "date_information\n", - "Q1075917\n", - "Q48751380\n", - "date_information\n", - "date_information\n", - "Q48674142\n", - "date_information\n", - "Q48733721\n", - "Q48733721\n", - "Q77906097\n", - "Q58411258\n", - "Q49700441\n", - "Q26259702\n", - "date_information\n", - "Q62894817\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47406390\n", - "date_information\n", - "date_information\n", - "Q3986110\n", - "date_information\n", - "Q19887161\n", - "Q64768142\n", - "date_information\n", - "date_information\n", - "Q66069164\n", - "date_information\n", - "Q28451742\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48838154\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48673974\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15689850\n", - "date_information\n", - "date_information\n", - "Q53775843\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62278093\n", - "Q4332349\n", - "Q65032875\n", - "date_information\n", - "Q1171105\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1061849\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62076011\n", - "date_information\n", - "Q48835820\n", - "date_information\n", - "Q67207071\n", - "date_information\n", - "Q22043860\n", - "date_information\n", - "Q62277630\n", - "date_information\n", - "date_information\n", - "Q65032623\n", - "date_information\n", - "date_information\n", - "Q56542193\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19887159\n", - "Q60286853\n", - "Q51879718\n", - "Q64006322\n", - "Q64006322\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55099304\n", - "date_information\n", - "date_information\n", - "Q64759224\n", - "Q61951885\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17082944\n", - "date_information\n", - "date_information\n", - "Q54858959\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60790967\n", - "date_information\n", - "date_information\n", - "Q77894162\n", - "Q2887372\n", - "Q2887372\n", - "Q1764531\n", - "date_information\n", - "Q16011979\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47500789\n", - "Q59707291\n", - "date_information\n", - "date_information\n", - "Q51930071\n", - "Q63437738\n", - "Q60748770\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q56274620\n", - "Q55634129\n", - "Q1498482\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17080705\n", - "date_information\n", - "Q55818691\n", - "date_information\n", - "date_information\n", - "Q48773837\n", - "Q48773837\n", - "date_information\n", - "Q1219865\n", - "date_information\n", - "Q55605990\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4392552\n", - "Q15633656\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56241156\n", - "date_information\n", - "Q1056692\n", - "Q60738619\n", - "Q47500754\n", - "date_information\n", - "date_information\n", - "Q66309770\n", - "date_information\n", - "Q58411015\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55637246\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768138\n", - "date_information\n", - "Q48811567\n", - "date_information\n", - "Q60743744\n", - "date_information\n", - "Q1052200\n", - "date_information\n", - "Q27\n", - "Q64768541\n", - "date_information\n", - "Q55633790\n", - "Q47383533\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62073524\n", - "date_information\n", - "Q14579154\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q6966375\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15681388\n", - "Q60637508\n", - "Q75261292\n", - "date_information\n", - "Q3567729\n", - "date_information\n", - "date_information\n", - "Q1145625\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55636360\n", - "date_information\n", - "date_information\n", - "Q55609995\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58815093\n", - "date_information\n", - "Q60738390\n", - "Q30609900\n", - "date_information\n", - "Q69741530\n", - "Q69741530\n", - "date_information\n", - "Q30887900\n", - "Q48674017\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48313997\n", - "date_information\n", - "Q64759221\n", - "Q3680696\n", - "Q3680696\n", - "Q1060166\n", - "Q58814954\n", - "date_information\n", - "Q48671641\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4532600\n", - "Q38251511\n", - "Q4079612\n", - "date_information\n", - "date_information\n", - "Q74192895\n", - "Q2371446\n", - "Q2371446\n", - "date_information\n", - "Q64368634\n", - "date_information\n", - "date_information\n", - "Q24041220\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63969269\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12473281\n", - "date_information\n", - "date_information\n", - "Q25610559\n", - "Q19760848\n", - "Q56280077\n", - "Q55614292\n", - "date_information\n", - "date_information\n", - "Q18923795\n", - "Q4325281\n", - "Q4207635\n", - "Q55564530\n", - "Q53721253\n", - "Q65963552\n", - "Q1618265\n", - "date_information\n", - "Q79212167\n", - "date_information\n", - "Q56274942\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56562348\n", - "date_information\n", - "Q27535874\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61450772\n", - "date_information\n", - "Q17558583\n", - "Q57697775\n", - "Q57017331\n", - "Q43528373\n", - "Q57017331\n", - "Q4059278\n", - "date_information\n", - "date_information\n", - "Q55635089\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55983731\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63127116\n", - "Q63127116\n", - "date_information\n", - "date_information\n", - "Q48673895\n", - "date_information\n", - "date_information\n", - "Q64338238\n", - "Q65068630\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4199883\n", - "date_information\n", - "Q64768613\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55762022\n", - "Q74192790\n", - "Q64460046\n", - "date_information\n", - "date_information\n", - "Q60738223\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55635194\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1982853\n", - "Q56692222\n", - "date_information\n", - "Q48733583\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75290417\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12978338\n", - "date_information\n", - "date_information\n", - "Q65924931\n", - "Q56906869\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4131389\n", - "Q55639353\n", - "Q48673975\n", - "Q12164415\n", - "Q26209116\n", - "Q21342433\n", - "date_information\n", - "Q60738009\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56692211\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q3744706\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738839\n", - "Q62277511\n", - "date_information\n", - "Q48700600\n", - "Q12916506\n", - "date_information\n", - "date_information\n", - "Q63183580\n", - "Q2348753\n", - "date_information\n", - "Q18340953\n", - "date_information\n", - "Q21979536\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54958943\n", - "date_information\n", - "Q7312\n", - "Q7312\n", - "date_information\n", - "date_information\n", - "Q12868778\n", - "Q4277663\n", - "date_information\n", - "Q62595549\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60220916\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50649913\n", - "date_information\n", - "Q64157092\n", - "date_information\n", - "date_information\n", - "Q60745425\n", - "date_information\n", - "date_information\n", - "Q4532705\n", - "date_information\n", - "Q21428011\n", - "date_information\n", - "Q56277505\n", - "Q75292970\n", - "date_information\n", - "Q28064471\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25508932\n", - "date_information\n", - "Q28434500\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3874453\n", - "date_information\n", - "date_information\n", - "Q21672950\n", - "date_information\n", - "Q17085747\n", - "date_information\n", - "Q57015954\n", - "date_information\n", - "Q56086939\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56736617\n", - "date_information\n", - "date_information\n", - "Q56452567\n", - "date_information\n", - "Q48817737\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55633925\n", - "date_information\n", - "Q54888784\n", - "Q73537536\n", - "date_information\n", - "Q18518927\n", - "Q18518927\n", - "Q2205928\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q7262511\n", - "date_information\n", - "Q27\n", - "Q2996959\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50649636\n", - "date_information\n", - "Q64768437\n", - "Q56565719\n", - "Q4233137\n", - "Q54622178\n", - "date_information\n", - "Q3998173\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q10923616\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55632806\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63924389\n", - "date_information\n", - "Q2047348\n", - "Q62277294\n", - "Q27\n", - "Q47480069\n", - "date_information\n", - "date_information\n", - "Q3988825\n", - "Q17478844\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3931523\n", - "date_information\n", - "Q62761144\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4349877\n", - "Q63994420\n", - "Q56703414\n", - "date_information\n", - "date_information\n", - "Q61806618\n", - "Q50649803\n", - "Q60737546\n", - "Q2943549\n", - "Q15052901\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12986578\n", - "date_information\n", - "date_information\n", - "Q55590165\n", - "date_information\n", - "Q17107235\n", - "Q17107235\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3520282\n", - "date_information\n", - "Q7295833\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52161645\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66305419\n", - "date_information\n", - "date_information\n", - "Q57373559\n", - "Q59304918\n", - "Q39078731\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q71037823\n", - "date_information\n", - "date_information\n", - "Q2861452\n", - "Q41438112\n", - "date_information\n", - "Q60576656\n", - "date_information\n", - "Q43901166\n", - "date_information\n", - "Q72426584\n", - "Q47894595\n", - "date_information\n", - "date_information\n", - "Q1127911\n", - "date_information\n", - "date_information\n", - "Q19794079\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15689444\n", - "date_information\n", - "date_information\n", - "Q57018894\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30749736\n", - "date_information\n", - "Q4462643\n", - "date_information\n", - "Q63183580\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61450788\n", - "date_information\n", - "date_information\n", - "Q68098810\n", - "Q68098810\n", - "Q77893670\n", - "date_information\n", - "Q65048227\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21103553\n", - "date_information\n", - "date_information\n", - "Q8051197\n", - "date_information\n", - "date_information\n", - "Q55613549\n", - "Q30880265\n", - "date_information\n", - "Q1065386\n", - "date_information\n", - "date_information\n", - "Q56063981\n", - "date_information\n", - "date_information\n", - "Q47500713\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q8849941\n", - "Q63923963\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24729376\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58814779\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16134165\n", - "date_information\n", - "Q59725715\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50389157\n", - "Q5890997\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51874967\n", - "Q48674354\n", - "date_information\n", - "Q10506677\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q43303197\n", - "Q55390545\n", - "date_information\n", - "date_information\n", - "Q60852120\n", - "Q15971720\n", - "Q15971720\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18244319\n", - "Q66764371\n", - "date_information\n", - "date_information\n", - "Q60738328\n", - "Q2940176\n", - "Q64460118\n", - "Q56524185\n", - "date_information\n", - "date_information\n", - "Q48815868\n", - "Q60793315\n", - "Q59725609\n", - "date_information\n", - "date_information\n", - "Q76488917\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66426745\n", - "Q19960535\n", - "Q19960535\n", - "date_information\n", - "date_information\n", - "Q63398330\n", - "Q60738189\n", - "Q54020785\n", - "date_information\n", - "Q12932196\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50821705\n", - "Q55635433\n", - "date_information\n", - "Q76273253\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56291167\n", - "date_information\n", - "Q52145034\n", - "date_information\n", - "date_information\n", - "Q12215667\n", - "date_information\n", - "Q51115846\n", - "date_information\n", - "date_information\n", - "Q16017012\n", - "Q56736603\n", - "date_information\n", - "Q5663147\n", - "Q12009607\n", - "date_information\n", - "Q49169290\n", - "Q3113471\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63450237\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q48802797\n", - "Q52715809\n", - "date_information\n", - "Q15706485\n", - "date_information\n", - "Q19329824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65060125\n", - "Q51754663\n", - "Q51754663\n", - "Q65926251\n", - "Q53850082\n", - "Q54959190\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24483410\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62631119\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19916425\n", - "date_information\n", - "date_information\n", - "Q57305695\n", - "date_information\n", - "date_information\n", - "Q4470976\n", - "date_information\n", - "Q61851090\n", - "Q63119094\n", - "Q60737531\n", - "Q48314237\n", - "date_information\n", - "Q19329824\n", - "Q4293645\n", - "Q46606592\n", - "date_information\n", - "Q63450164\n", - "Q55635245\n", - "Q3858187\n", - "date_information\n", - "date_information\n", - "Q21904895\n", - "date_information\n", - "Q44491908\n", - "Q55394005\n", - "Q48838154\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63936568\n", - "Q59725690\n", - "date_information\n", - "date_information\n", - "Q60738541\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q53866621\n", - "date_information\n", - "date_information\n", - "Q67207178\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48674000\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16029045\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47044251\n", - "Q19907427\n", - "date_information\n", - "Q19848155\n", - "date_information\n", - "Q65090404\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65065339\n", - "date_information\n", - "date_information\n", - "Q3206523\n", - "Q1584799\n", - "Q1584799\n", - "date_information\n", - "date_information\n", - "Q64747722\n", - "date_information\n", - "Q56550947\n", - "Q60742477\n", - "date_information\n", - "date_information\n", - "Q47455556\n", - "Q70216239\n", - "date_information\n", - "Q69629123\n", - "date_information\n", - "Q56756921\n", - "date_information\n", - "Q64174957\n", - "date_information\n", - "Q64577348\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58814954\n", - "date_information\n", - "Q56599922\n", - "date_information\n", - "Q56341064\n", - "Q24324411\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1788143\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25610607\n", - "Q58310059\n", - "date_information\n", - "date_information\n", - "Q60738161\n", - "date_information\n", - "date_information\n", - "Q63565895\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52715530\n", - "Q4193723\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55392250\n", - "Q18527744\n", - "date_information\n", - "Q63700222\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47064364\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19613796\n", - "Q48317293\n", - "Q4470872\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19691592\n", - "date_information\n", - "date_information\n", - "Q66305400\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3987982\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77906352\n", - "date_information\n", - "date_information\n", - "Q59732835\n", - "Q56063528\n", - "Q47063963\n", - "Q57524498\n", - "Q38251747\n", - "date_information\n", - "Q51885874\n", - "date_information\n", - "Q65071983\n", - "date_information\n", - "date_information\n", - "Q15990861\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55638119\n", - "date_information\n", - "Q18527744\n", - "Q18527744\n", - "date_information\n", - "Q65924953\n", - "date_information\n", - "Q3549581\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3209538\n", - "date_information\n", - "Q62277324\n", - "date_information\n", - "Q51834363\n", - "Q24041220\n", - "Q61847417\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q45710096\n", - "date_information\n", - "date_information\n", - "Q48792062\n", - "date_information\n", - "Q60739768\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30766241\n", - "Q64768089\n", - "date_information\n", - "Q56241170\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2940176\n", - "date_information\n", - "Q23565012\n", - "Q64768509\n", - "Q12980738\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56692194\n", - "Q52151197\n", - "date_information\n", - "Q61911477\n", - "Q61911477\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66828793\n", - "date_information\n", - "Q63450143\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17478936\n", - "Q21768346\n", - "date_information\n", - "date_information\n", - "Q3206905\n", - "Q4231430\n", - "date_information\n", - "date_information\n", - "Q62277886\n", - "date_information\n", - "date_information\n", - "Q64768570\n", - "date_information\n", - "Q19329824\n", - "Q66764353\n", - "date_information\n", - "Q61951929\n", - "Q20672424\n", - "date_information\n", - "Q60738569\n", - "date_information\n", - "Q66663995\n", - "Q27\n", - "date_information\n", - "Q4019398\n", - "date_information\n", - "Q47455556\n", - "date_information\n", - "date_information\n", - "Q12913196\n", - "date_information\n", - "date_information\n", - "Q56275856\n", - "Q10368665\n", - "date_information\n", - "Q60575326\n", - "Q43476852\n", - "date_information\n", - "date_information\n", - "Q64768138\n", - "Q1285133\n", - "Q13041037\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60737591\n", - "Q63924459\n", - "date_information\n", - "Q64768308\n", - "date_information\n", - "Q59149494\n", - "Q24728729\n", - "Q65532105\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q64920706\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50649795\n", - "Q65087830\n", - "Q25505774\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q45838757\n", - "date_information\n", - "Q63198193\n", - "date_information\n", - "date_information\n", - "Q61747349\n", - "Q60745215\n", - "Q51836237\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q24336546\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55402704\n", - "Q1049832\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50650117\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20029149\n", - "date_information\n", - "Q2861646\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56703330\n", - "Q27\n", - "Q48671819\n", - "date_information\n", - "date_information\n", - "Q4087902\n", - "Q22043708\n", - "date_information\n", - "Q18398913\n", - "date_information\n", - "Q19847844\n", - "date_information\n", - "date_information\n", - "Q30901563\n", - "date_information\n", - "date_information\n", - "Q55603013\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54868287\n", - "date_information\n", - "Q58815150\n", - "Q48674292\n", - "date_information\n", - "date_information\n", - "Q19908105\n", - "date_information\n", - "Q62018331\n", - "Q65043142\n", - "Q65043142\n", - "Q65122755\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16123155\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12986338\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63398433\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q72219237\n", - "date_information\n", - "date_information\n", - "Q48791886\n", - "Q48791886\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55604143\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q8718447\n", - "Q54554846\n", - "Q54554846\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47500725\n", - "Q4324144\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q63119063\n", - "date_information\n", - "Q28527409\n", - "Q48671922\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60600320\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65076241\n", - "date_information\n", - "Q55603039\n", - "Q28407934\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768067\n", - "date_information\n", - "Q60853109\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4372982\n", - "Q456239\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47090778\n", - "Q66691283\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21338430\n", - "Q57901801\n", - "date_information\n", - "Q48837015\n", - "Q13110027\n", - "Q12989263\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738349\n", - "Q63871479\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q53844848\n", - "Q28839352\n", - "Q28839352\n", - "date_information\n", - "Q6206924\n", - "Q6206924\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17083476\n", - "date_information\n", - "date_information\n", - "Q16017012\n", - "date_information\n", - "Q59725766\n", - "Q54240121\n", - "date_information\n", - "date_information\n", - "Q48700026\n", - "Q48700026\n", - "date_information\n", - "date_information\n", - "Q61861582\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77894281\n", - "date_information\n", - "date_information\n", - "Q16679535\n", - "date_information\n", - "Q16135618\n", - "Q63568445\n", - "date_information\n", - "date_information\n", - "Q60738248\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56006321\n", - "date_information\n", - "date_information\n", - "Q45709616\n", - "date_information\n", - "date_information\n", - "Q60637469\n", - "Q64350350\n", - "Q64350350\n", - "Q55420238\n", - "Q7936508\n", - "Q7936508\n", - "Q48672695\n", - "date_information\n", - "date_information\n", - "Q6145612\n", - "date_information\n", - "Q64759208\n", - "date_information\n", - "date_information\n", - "Q7634567\n", - "date_information\n", - "Q48778903\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3719053\n", - "Q3719053\n", - "Q55624128\n", - "Q54864392\n", - "Q54864392\n", - "date_information\n", - "date_information\n", - "Q56283796\n", - "date_information\n", - "Q28407066\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q25505817\n", - "date_information\n", - "Q21484965\n", - "date_information\n", - "Q12284695\n", - "Q63968536\n", - "Q63968536\n", - "date_information\n", - "date_information\n", - "Q56703337\n", - "date_information\n", - "Q50280810\n", - "Q10542513\n", - "Q65065437\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27835274\n", - "Q55595776\n", - "date_information\n", - "Q54894974\n", - "Q65679602\n", - "date_information\n", - "date_information\n", - "Q58411662\n", - "Q57524491\n", - "Q4124302\n", - "Q12913776\n", - "Q27\n", - "Q60674735\n", - "date_information\n", - "Q28000916\n", - "date_information\n", - "Q61015509\n", - "Q55364056\n", - "Q28406674\n", - "date_information\n", - "date_information\n", - "Q4240259\n", - "Q76488790\n", - "Q65061242\n", - "Q56486741\n", - "Q16028462\n", - "date_information\n", - "Q14942560\n", - "Q3086265\n", - "Q3086265\n", - "date_information\n", - "date_information\n", - "Q24484582\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17081676\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q60738001\n", - "date_information\n", - "date_information\n", - "Q44506948\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4382036\n", - "Q54894996\n", - "date_information\n", - "Q62277511\n", - "Q51818072\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55379558\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3749740\n", - "Q3749740\n", - "Q48845168\n", - "date_information\n", - "date_information\n", - "Q63923963\n", - "date_information\n", - "Q51852910\n", - "Q56278501\n", - "date_information\n", - "date_information\n", - "Q16316783\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q24729316\n", - "date_information\n", - "Q60737786\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q9302591\n", - "Q9302591\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q70220807\n", - "date_information\n", - "Q56651031\n", - "Q75518257\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q51543325\n", - "date_information\n", - "Q12278958\n", - "date_information\n", - "Q21668126\n", - "Q62018325\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q13009165\n", - "date_information\n", - "date_information\n", - "Q65060213\n", - "date_information\n", - "date_information\n", - "Q15615535\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18411469\n", - "date_information\n", - "Q19760872\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48672584\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q56276985\n", - "date_information\n", - "date_information\n", - "Q59693825\n", - "date_information\n", - "date_information\n", - "Q23957301\n", - "date_information\n", - "Q48790272\n", - "Q63371558\n", - "Q51756473\n", - "Q48671393\n", - "date_information\n", - "Q62018336\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61867559\n", - "Q61867559\n", - "Q25203133\n", - "date_information\n", - "date_information\n", - "Q17632127\n", - "Q47485925\n", - "Q47485925\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q4860128\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12988245\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17539165\n", - "Q19329824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64759213\n", - "date_information\n", - "date_information\n", - "Q20712565\n", - "date_information\n", - "Q55637842\n", - "date_information\n", - "date_information\n", - "Q54020958\n", - "date_information\n", - "Q59725898\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58898819\n", - "Q58898846\n", - "Q58898846\n", - "Q63450143\n", - "date_information\n", - "date_information\n", - "Q51879925\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q13010497\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64577372\n", - "date_information\n", - "Q5747\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19461433\n", - "Q58411313\n", - "Q15717576\n", - "date_information\n", - "Q56026007\n", - "Q197331\n", - "Q197331\n", - "Q55635485\n", - "date_information\n", - "Q11012680\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18554460\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q23305110\n", - "date_information\n", - "date_information\n", - "Q52715587\n", - "Q4198706\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50280802\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q9265906\n", - "date_information\n", - "Q48739505\n", - "Q25508381\n", - "Q48550528\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q39060677\n", - "date_information\n", - "Q3810224\n", - "Q24324379\n", - "Q56452535\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65077628\n", - "date_information\n", - "date_information\n", - "Q64355175\n", - "date_information\n", - "Q13478702\n", - "Q55614158\n", - "date_information\n", - "date_information\n", - "Q43463849\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30727118\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12926922\n", - "date_information\n", - "Q60737774\n", - "date_information\n", - "Q1075755\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2972912\n", - "date_information\n", - "date_information\n", - "Q1055275\n", - "date_information\n", - "date_information\n", - "Q6836288\n", - "Q47500790\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q43401863\n", - "date_information\n", - "Q17080556\n", - "date_information\n", - "Q50650337\n", - "date_information\n", - "date_information\n", - "Q65031957\n", - "date_information\n", - "Q60738340\n", - "Q1070459\n", - "Q16989678\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q73538385\n", - "Q55605100\n", - "Q60767033\n", - "Q48673919\n", - "date_information\n", - "Q63936568\n", - "Q17083476\n", - "Q17562822\n", - "Q5990904\n", - "date_information\n", - "Q56816865\n", - "Q56678472\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62073359\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768096\n", - "Q1068903\n", - "Q39075339\n", - "date_information\n", - "date_information\n", - "Q3635242\n", - "Q3635242\n", - "date_information\n", - "Q65066171\n", - "date_information\n", - "Q50650326\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77894866\n", - "date_information\n", - "date_information\n", - "Q55641567\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55946142\n", - "date_information\n", - "Q979254\n", - "Q979254\n", - "date_information\n", - "Q65081771\n", - "date_information\n", - "Q65118726\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47455556\n", - "date_information\n", - "date_information\n", - "Q18910297\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q54817993\n", - "date_information\n", - "Q61759649\n", - "Q65046237\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4349877\n", - "date_information\n", - "Q63994407\n", - "date_information\n", - "date_information\n", - "Q63871479\n", - "Q17082384\n", - "date_information\n", - "Q57246581\n", - "Q80117984\n", - "Q43451770\n", - "date_information\n", - "date_information\n", - "Q4168689\n", - "Q57246581\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q603711\n", - "Q62018336\n", - "Q603711\n", - "Q56277721\n", - "date_information\n", - "date_information\n", - "Q30901011\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56071279\n", - "Q9265906\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1120892\n", - "Q21752640\n", - "Q64768601\n", - "date_information\n", - "Q75279216\n", - "Q75341911\n", - "date_information\n", - "Q51278957\n", - "Q55602989\n", - "Q49758553\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q20463710\n", - "Q25505760\n", - "date_information\n", - "date_information\n", - "Q16016876\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738167\n", - "Q62075999\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q7245965\n", - "date_information\n", - "Q17479256\n", - "date_information\n", - "date_information\n", - "Q51233362\n", - "Q18554460\n", - "Q50623892\n", - "Q27969667\n", - "Q44865478\n", - "Q2287587\n", - "Q1853096\n", - "Q1853096\n", - "Q2827027\n", - "Q27\n", - "Q17081754\n", - "date_information\n", - "date_information\n", - "Q56664002\n", - "Q56664002\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52004168\n", - "date_information\n", - "Q73550176\n", - "Q63119063\n", - "Q64026414\n", - "date_information\n", - "date_information\n", - "Q15689975\n", - "Q42279997\n", - "date_information\n", - "date_information\n", - "Q66098678\n", - "date_information\n", - "Q54868308\n", - "Q55634452\n", - "Q57418840\n", - "Q57418840\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56064452\n", - "Q56199338\n", - "Q16988670\n", - "date_information\n", - "Q17081590\n", - "Q19845028\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75518787\n", - "Q17080124\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q59733907\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19915851\n", - "Q4459376\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64022059\n", - "date_information\n", - "Q37609273\n", - "date_information\n", - "date_information\n", - "Q46625720\n", - "date_information\n", - "Q65677542\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48671397\n", - "date_information\n", - "Q27\n", - "Q3421105\n", - "Q3421105\n", - "Q24907674\n", - "Q65045904\n", - "date_information\n", - "Q61830133\n", - "Q61830133\n", - "date_information\n", - "date_information\n", - "Q16570900\n", - "Q48674765\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12916211\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q38011705\n", - "Q56692217\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64031533\n", - "date_information\n", - "Q27786016\n", - "date_information\n", - "date_information\n", - "Q1336307\n", - "Q1336307\n", - "Q51845845\n", - "date_information\n", - "Q60738417\n", - "Q12987301\n", - "Q3197182\n", - "Q60773660\n", - "date_information\n", - "date_information\n", - "Q28457459\n", - "date_information\n", - "Q11014572\n", - "Q3192937\n", - "Q105959\n", - "date_information\n", - "Q52267697\n", - "Q64768601\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62018324\n", - "date_information\n", - "Q60737570\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q30901566\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19258530\n", - "date_information\n", - "Q53098702\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q16029045\n", - "date_information\n", - "Q21336464\n", - "date_information\n", - "Q16675670\n", - "date_information\n", - "Q30634277\n", - "Q63348188\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60737884\n", - "Q4019811\n", - "date_information\n", - "Q62081531\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12913150\n", - "Q67207072\n", - "Q1054889\n", - "Q2569449\n", - "date_information\n", - "Q55632402\n", - "Q23059522\n", - "Q23059522\n", - "date_information\n", - "Q55615148\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48758493\n", - "date_information\n", - "date_information\n", - "Q56064161\n", - "date_information\n", - "Q55612161\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16304177\n", - "date_information\n", - "date_information\n", - "Q56279396\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q46835085\n", - "Q1563140\n", - "Q1563140\n", - "date_information\n", - "Q7312\n", - "Q7312\n", - "date_information\n", - "Q55403066\n", - "date_information\n", - "Q55633241\n", - "Q24529636\n", - "Q24529636\n", - "date_information\n", - "date_information\n", - "Q53577577\n", - "Q53007522\n", - "Q53007522\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768127\n", - "date_information\n", - "Q25490712\n", - "date_information\n", - "date_information\n", - "Q48771076\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55632310\n", - "date_information\n", - "Q55633790\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48674090\n", - "date_information\n", - "Q20202440\n", - "date_information\n", - "date_information\n", - "Q63565881\n", - "date_information\n", - "date_information\n", - "Q3336535\n", - "date_information\n", - "Q65080880\n", - "date_information\n", - "Q65032636\n", - "Q60575326\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55081824\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65919093\n", - "date_information\n", - "Q55611328\n", - "date_information\n", - "date_information\n", - "Q30901595\n", - "date_information\n", - "Q30901011\n", - "date_information\n", - "Q3680696\n", - "Q3680696\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q60222985\n", - "date_information\n", - "Q55603751\n", - "date_information\n", - "Q4092188\n", - "date_information\n", - "Q56703344\n", - "Q60738712\n", - "date_information\n", - "Q3523359\n", - "date_information\n", - "Q28222734\n", - "date_information\n", - "Q39070959\n", - "Q63859758\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "Q54935610\n", - "date_information\n", - "date_information\n", - "Q1054858\n", - "Q55907451\n", - "Q54864392\n", - "Q54864392\n", - "date_information\n", - "date_information\n", - "Q11157534\n", - "Q62595428\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17084224\n", - "date_information\n", - "date_information\n", - "Q56254328\n", - "date_information\n", - "Q16016291\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q62018331\n", - "Q65043142\n", - "Q65043142\n", - "date_information\n", - "Q58843996\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28434507\n", - "Q4520010\n", - "Q49822556\n", - "date_information\n", - "Q57524505\n", - "date_information\n", - "date_information\n", - "Q46996529\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56279507\n", - "Q75321363\n", - "date_information\n", - "Q77894130\n", - "Q73536578\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56452591\n", - "Q52781693\n", - "Q27\n", - "Q56452620\n", - "date_information\n", - "date_information\n", - "Q12988455\n", - "date_information\n", - "date_information\n", - "Q3989046\n", - "Q55394008\n", - "date_information\n", - "Q64768025\n", - "Q77892368\n", - "Q58815211\n", - "Q47455556\n", - "Q28509222\n", - "Q19907129\n", - "Q4061838\n", - "Q56274092\n", - "Q12156257\n", - "date_information\n", - "date_information\n", - "Q65032618\n", - "date_information\n", - "Q65117469\n", - "date_information\n", - "date_information\n", - "Q19349761\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19907917\n", - "Q24729099\n", - "Q63614867\n", - "date_information\n", - "Q23771092\n", - "Q56878204\n", - "Q60738665\n", - "Q56868186\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q774365\n", - "Q8253991\n", - "Q56276915\n", - "date_information\n", - "Q2035210\n", - "date_information\n", - "Q75256284\n", - "date_information\n", - "Q65075111\n", - "Q56240003\n", - "date_information\n", - "Q48988512\n", - "Q4441321\n", - "Q75336320\n", - "date_information\n", - "date_information\n", - "Q21081425\n", - "Q10542513\n", - "Q61740801\n", - "date_information\n", - "Q31080455\n", - "Q75269403\n", - "Q18708960\n", - "Q50280807\n", - "Q27\n", - "Q11928049\n", - "Q75574432\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3881205\n", - "date_information\n", - "Q59725700\n", - "Q61268438\n", - "date_information\n", - "Q65059746\n", - "date_information\n", - "date_information\n", - "Q18546743\n", - "Q75261325\n", - "Q3213381\n", - "Q55818699\n", - "Q23957665\n", - "Q58041648\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48671900\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q22119552\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q61955952\n", - "Q3731987\n", - "date_information\n", - "date_information\n", - "Q56284238\n", - "Q55163823\n", - "Q55625451\n", - "Q75248108\n", - "Q3175522\n", - "Q64627111\n", - "Q4491432\n", - "Q6560644\n", - "date_information\n", - "Q4076943\n", - "Q1735461\n", - "date_information\n", - "Q16373255\n", - "date_information\n", - "Q60671508\n", - "Q51278866\n", - "date_information\n", - "date_information\n", - "Q75299375\n", - "Q63924441\n", - "Q51026837\n", - "Q51835369\n", - "Q73537979\n", - "date_information\n", - "Q55552441\n", - "date_information\n", - "Q7312\n", - "Q7312\n", - "Q4345832\n", - "Q9312628\n", - "Q55390128\n", - "Q53736577\n", - "date_information\n", - "date_information\n", - "Q75251661\n", - "date_information\n", - "Q27\n", - "Q883568\n", - "date_information\n", - "Q7237205\n", - "Q75388243\n", - "date_information\n", - "date_information\n", - "Q30668079\n", - "date_information\n", - "date_information\n", - "Q63348173\n", - "date_information\n", - "Q19691746\n", - "Q24728138\n", - "date_information\n", - "Q13105441\n", - "date_information\n", - "Q110521\n", - "Q55590165\n", - "Q2515665\n", - "Q2515665\n", - "Q75384531\n", - "Q55607104\n", - "date_information\n", - "date_information\n", - "Q47484410\n", - "date_information\n", - "Q64577679\n", - "Q75304204\n", - "Q21183741\n", - "Q1464130\n", - "date_information\n", - "Q15278116\n", - "Q48671955\n", - "Q331352\n", - "Q331352\n", - "date_information\n", - "Q18234383\n", - "Q67191054\n", - "Q57821032\n", - "Q55635336\n", - "Q27\n", - "Q1147037\n", - "Q4066550\n", - "date_information\n", - "Q21503271\n", - "Q60461858\n", - "Q27\n", - "Q75246969\n", - "Q862775\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3292912\n", - "Q3292912\n", - "date_information\n", - "Q3989046\n", - "date_information\n", - "Q15123159\n", - "Q55610834\n", - "Q1012024\n", - "Q56276996\n", - "Q55603668\n", - "date_information\n", - "Q60737725\n", - "Q56248006\n", - "Q56248006\n", - "Q52063188\n", - "Q21035652\n", - "date_information\n", - "Q60744747\n", - "date_information\n", - "date_information\n", - "Q65060125\n", - "Q26884129\n", - "date_information\n", - "Q44226364\n", - "Q44226364\n", - "date_information\n", - "date_information\n", - "Q55629138\n", - "Q56561387\n", - "date_information\n", - "date_information\n", - "Q51515927\n", - "Q51031099\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4380867\n", - "Q48673975\n", - "date_information\n", - "Q3749740\n", - "Q3749740\n", - "Q73536712\n", - "date_information\n", - "Q2554051\n", - "Q30693497\n", - "Q188605\n", - "Q55625236\n", - "date_information\n", - "date_information\n", - "Q64018069\n", - "Q48674160\n", - "Q48671762\n", - "date_information\n", - "Q67442788\n", - "date_information\n", - "Q19760881\n", - "Q60741350\n", - "date_information\n", - "date_information\n", - "Q3431361\n", - "Q77903089\n", - "date_information\n", - "date_information\n", - "Q1245637\n", - "date_information\n", - "Q13652012\n", - "Q25739739\n", - "Q14933003\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q77895574\n", - "Q50828546\n", - "date_information\n", - "Q75467047\n", - "Q75271130\n", - "Q21183854\n", - "Q63934645\n", - "date_information\n", - "date_information\n", - "Q2461403\n", - "Q1996080\n", - "Q75276810\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q70895877\n", - "date_information\n", - "Q15706008\n", - "Q76074470\n", - "Q48674657\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65559778\n", - "Q3290274\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4382036\n", - "Q63802341\n", - "Q59776759\n", - "Q61483064\n", - "Q3086265\n", - "Q3086265\n", - "date_information\n", - "date_information\n", - "Q1507650\n", - "Q1665346\n", - "Q1665346\n", - "Q1743888\n", - "Q897472\n", - "Q306928\n", - "Q66305414\n", - "Q63259135\n", - "date_information\n", - "Q62736780\n", - "Q60737739\n", - "date_information\n", - "date_information\n", - "Q64996711\n", - "Q64996711\n", - "Q22302425\n", - "Q60737531\n", - "date_information\n", - "Q75268886\n", - "Q22344308\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28064524\n", - "date_information\n", - "Q74460881\n", - "Q54868269\n", - "Q1147037\n", - "Q75485298\n", - "Q1147037\n", - "date_information\n", - "date_information\n", - "Q50825810\n", - "Q63986495\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q47451374\n", - "Q664627\n", - "Q44077104\n", - "Q75399987\n", - "date_information\n", - "Q18511263\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64606091\n", - "Q3336838\n", - "Q3336838\n", - "date_information\n", - "Q76186239\n", - "Q30591334\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q883568\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60750630\n", - "Q60750679\n", - "Q60750679\n", - "Q75872063\n", - "date_information\n", - "Q56648565\n", - "date_information\n", - "Q3429285\n", - "Q63565878\n", - "Q61911606\n", - "Q55641895\n", - "Q75908259\n", - "Q57398740\n", - "date_information\n", - "Q59617591\n", - "Q50821742\n", - "Q56703415\n", - "date_information\n", - "date_information\n", - "Q50649699\n", - "Q75206451\n", - "date_information\n", - "date_information\n", - "Q55614737\n", - "Q56736600\n", - "date_information\n", - "date_information\n", - "Q47969761\n", - "date_information\n", - "date_information\n", - "Q26722753\n", - "date_information\n", - "Q2387732\n", - "Q47293554\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q14327504\n", - "Q14327504\n", - "date_information\n", - "date_information\n", - "Q55659359\n", - "Q60565417\n", - "Q5769416\n", - "date_information\n", - "Q16017234\n", - "Q75278625\n", - "date_information\n", - "Q1070575\n", - "Q3432499\n", - "date_information\n", - "date_information\n", - "Q56275862\n", - "Q43476870\n", - "Q16761120\n", - "Q20085644\n", - "date_information\n", - "Q13677685\n", - "Q13677685\n", - "date_information\n", - "Q75566771\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q4020590\n", - "Q54958420\n", - "date_information\n", - "date_information\n", - "Q9005\n", - "Q65677546\n", - "Q48700092\n", - "Q55634490\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q37553201\n", - "Q3176228\n", - "date_information\n", - "Q3429837\n", - "Q48674540\n", - "Q5675398\n", - "date_information\n", - "date_information\n", - "Q700380\n", - "Q73536842\n", - "Q188605\n", - "date_information\n", - "Q12867338\n", - "Q60738232\n", - "date_information\n", - "date_information\n", - "Q56064161\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q15622114\n", - "Q65051148\n", - "date_information\n", - "date_information\n", - "Q75397190\n", - "date_information\n", - "Q55391785\n", - "date_information\n", - "Q2662932\n", - "date_information\n", - "date_information\n", - "Q73536921\n", - "Q9005\n", - "Q55640205\n", - "Q56678809\n", - "Q50888343\n", - "date_information\n", - "Q48671308\n", - "Q16605834\n", - "Q60737799\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q7074764\n", - "Q52715635\n", - "Q3235756\n", - "date_information\n", - "Q56045021\n", - "Q56225996\n", - "Q58814779\n", - "Q51031282\n", - "date_information\n", - "Q55616601\n", - "date_information\n", - "Q75256284\n", - "date_information\n", - "Q49746290\n", - "Q19906881\n", - "Q4093441\n", - "date_information\n", - "Q376669\n", - "date_information\n", - "Q18912995\n", - "Q75885105\n", - "Q7128861\n", - "Q60791225\n", - "Q7830594\n", - "Q64460079\n", - "Q2865169\n", - "date_information\n", - "Q16832849\n", - "Q4956744\n", - "date_information\n", - "Q25508309\n", - "Q4318960\n", - "Q76344484\n", - "Q3818605\n", - "Q50804871\n", - "date_information\n", - "Q9005\n", - "date_information\n", - "Q25563078\n", - "Q1895150\n", - "Q55258969\n", - "Q60738268\n", - "Q15649511\n", - "Q13811044\n", - "date_information\n", - "Q47063735\n", - "Q4386849\n", - "Q75253659\n", - "Q65090316\n", - "Q18730165\n", - "Q11923812\n", - "date_information\n", - "Q2134675\n", - "date_information\n", - "Q774365\n", - "date_information\n", - "Q56062284\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47498866\n", - "date_information\n", - "Q55762032\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q2150379\n", - "date_information\n", - "Q55635513\n", - "date_information\n", - "date_information\n", - "Q48674507\n", - "Q15629331\n", - "date_information\n", - "Q13811044\n", - "date_information\n", - "Q164518\n", - "date_information\n", - "Q2387732\n", - "date_information\n", - "Q56223448\n", - "Q75250983\n", - "date_information\n", - "Q56886907\n", - "Q114953\n", - "Q42298769\n", - "Q66472629\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768017\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47167138\n", - "Q1870565\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "Q63282851\n", - "Q63924358\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q327356\n", - "Q17462238\n", - "date_information\n", - "date_information\n", - "Q24055383\n", - "date_information\n", - "Q5261141\n", - "Q7312\n", - "Q7312\n", - "date_information\n", - "date_information\n", - "Q12201526\n", - "Q55950925\n", - "date_information\n", - "Q2833398\n", - "date_information\n", - "Q50828038\n", - "date_information\n", - "Q4078100\n", - "Q11402101\n", - "date_information\n", - "date_information\n", - "Q51070711\n", - "Q63951591\n", - "date_information\n", - "Q67171439\n", - "Q66023775\n", - "Q66023775\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50411737\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75874200\n", - "Q59474327\n", - "date_information\n", - "Q2944916\n", - "Q12286002\n", - "Q51844371\n", - "Q51844371\n", - "Q2272984\n", - "date_information\n", - "Q16761120\n", - "date_information\n", - "date_information\n", - "Q1758660\n", - "Q55630320\n", - "Q55907550\n", - "Q48835993\n", - "Q65677544\n", - "date_information\n", - "date_information\n", - "Q57584319\n", - "Q75885107\n", - "date_information\n", - "Q25563078\n", - "Q27\n", - "Q7312\n", - "Q7312\n", - "Q51026837\n", - "Q55590142\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q76250886\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75341381\n", - "date_information\n", - "Q36705119\n", - "Q164518\n", - "Q3603202\n", - "date_information\n", - "date_information\n", - "Q2963134\n", - "Q2891591\n", - "Q2891591\n", - "Q3358612\n", - "Q64414258\n", - "Q52715740\n", - "date_information\n", - "Q56274013\n", - "date_information\n", - "Q3191819\n", - "Q2884747\n", - "Q2884747\n", - "Q3358612\n", - "Q27\n", - "Q55081826\n", - "date_information\n", - "Q59725655\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56290550\n", - "Q56434717\n", - "Q48804187\n", - "Q64010219\n", - "Q64789566\n", - "Q55634490\n", - "date_information\n", - "Q3613953\n", - "Q3613953\n", - "date_information\n", - "Q42322839\n", - "Q5921305\n", - "Q60738295\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60745846\n", - "date_information\n", - "Q8253991\n", - "date_information\n", - "Q51070711\n", - "date_information\n", - "Q48965171\n", - "Q65117495\n", - "Q65117495\n", - "date_information\n", - "date_information\n", - "Q60738142\n", - "Q55950746\n", - "Q15456810\n", - "Q10274143\n", - "date_information\n", - "Q55443439\n", - "Q60310822\n", - "Q60310822\n", - "Q58411513\n", - "Q45710417\n", - "date_information\n", - "date_information\n", - "Q3978256\n", - "Q3978256\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28966251\n", - "date_information\n", - "Q23771092\n", - "Q56308962\n", - "Q55081822\n", - "Q75574432\n", - "Q55070339\n", - "date_information\n", - "date_information\n", - "Q75261282\n", - "date_information\n", - "date_information\n", - "Q50848188\n", - "date_information\n", - "Q74586939\n", - "Q58008825\n", - "Q58008825\n", - "date_information\n", - "Q15854327\n", - "date_information\n", - "date_information\n", - "Q44727126\n", - "Q55391524\n", - "Q75776327\n", - "Q4580282\n", - "date_information\n", - "Q50143448\n", - "Q58411738\n", - "Q56064134\n", - "Q56064134\n", - "date_information\n", - "Q6292594\n", - "Q48965262\n", - "Q18576842\n", - "date_information\n", - "Q44492116\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q34134216\n", - "Q47546716\n", - "Q47546716\n", - "date_information\n", - "Q3660545\n", - "Q56240003\n", - "Q46230954\n", - "Q2137291\n", - "date_information\n", - "Q27\n", - "Q67072745\n", - "date_information\n", - "Q30591104\n", - "Q75560229\n", - "date_information\n", - "Q33170323\n", - "date_information\n", - "date_information\n", - "Q39050904\n", - "date_information\n", - "date_information\n", - "Q73536750\n", - "Q54861541\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55623515\n", - "Q75634523\n", - "Q52159104\n", - "date_information\n", - "Q61466220\n", - "Q12863348\n", - "Q16134448\n", - "Q16134448\n", - "Q74448919\n", - "Q60737559\n", - "Q15278116\n", - "date_information\n", - "date_information\n", - "Q75250845\n", - "date_information\n", - "Q48809669\n", - "date_information\n", - "date_information\n", - "Q61450786\n", - "date_information\n", - "date_information\n", - "Q75243284\n", - "date_information\n", - "date_information\n", - "Q24728388\n", - "Q21993842\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q52715703\n", - "date_information\n", - "date_information\n", - "Q24950627\n", - "Q24950627\n", - "Q5056642\n", - "date_information\n", - "Q21000712\n", - "Q21000712\n", - "date_information\n", - "Q75252209\n", - "Q44077095\n", - "Q64338145\n", - "date_information\n", - "date_information\n", - "Q15278101\n", - "date_information\n", - "date_information\n", - "Q3010170\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47035225\n", - "Q75282866\n", - "date_information\n", - "Q61851107\n", - "Q20792485\n", - "date_information\n", - "Q20891257\n", - "Q15062295\n", - "Q15062295\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q22275256\n", - "Q55599829\n", - "Q47200677\n", - "date_information\n", - "date_information\n", - "Q50168929\n", - "Q12460488\n", - "Q67497715\n", - "Q63183577\n", - "Q65058944\n", - "Q188605\n", - "Q63380908\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17081338\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q19886995\n", - "Q75783716\n", - "Q57779969\n", - "date_information\n", - "Q48815173\n", - "Q2905436\n", - "Q75567379\n", - "date_information\n", - "date_information\n", - "Q57150126\n", - "Q60738612\n", - "date_information\n", - "Q56604162\n", - "Q75388243\n", - "Q61954782\n", - "date_information\n", - "Q63802231\n", - "Q56761838\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q5862523\n", - "Q52151197\n", - "Q2879099\n", - "Q12720302\n", - "date_information\n", - "Q27\n", - "Q60738015\n", - "Q39758953\n", - "Q39758953\n", - "Q55631639\n", - "date_information\n", - "Q48672679\n", - "Q75420141\n", - "Q27\n", - "date_information\n", - "Q60036950\n", - "date_information\n", - "Q5623543\n", - "Q20075808\n", - "date_information\n", - "Q30636729\n", - "date_information\n", - "date_information\n", - "Q56281346\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q327356\n", - "Q15728101\n", - "date_information\n", - "date_information\n", - "Q25218388\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60738495\n", - "Q21707663\n", - "date_information\n", - "Q63924517\n", - "date_information\n", - "Q254200\n", - "Q23955378\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64947919\n", - "date_information\n", - "Q65080698\n", - "Q76276445\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18419557\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q44252992\n", - "Q65049717\n", - "date_information\n", - "Q59656542\n", - "Q40745153\n", - "Q10563008\n", - "date_information\n", - "date_information\n", - "Q75338760\n", - "date_information\n", - "date_information\n", - "Q73792248\n", - "date_information\n", - "Q59240227\n", - "Q47500739\n", - "date_information\n", - "Q27\n", - "Q48671152\n", - "date_information\n", - "Q11904222\n", - "Q60750745\n", - "Q2944916\n", - "date_information\n", - "Q612684\n", - "Q1665346\n", - "Q1665346\n", - "Q1743888\n", - "Q5764567\n", - "date_information\n", - "Q60169300\n", - "Q60169300\n", - "date_information\n", - "Q61890364\n", - "Q65118239\n", - "Q63977741\n", - "Q48770601\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3635242\n", - "Q3635242\n", - "Q59513147\n", - "Q75321315\n", - "Q3471889\n", - "Q25505901\n", - "Q25508765\n", - "date_information\n", - "Q2836351\n", - "Q56225996\n", - "Q327356\n", - "Q2619334\n", - "Q51756971\n", - "Q47793419\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16557779\n", - "Q766543\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q64768480\n", - "Q75280625\n", - "date_information\n", - "date_information\n", - "Q62589387\n", - "Q60467328\n", - "Q60467328\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q60048295\n", - "Q75250730\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65924945\n", - "date_information\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75273751\n", - "Q64018051\n", - "Q51280942\n", - "Q51276514\n", - "Q51276514\n", - "date_information\n", - "Q7334238\n", - "Q7334238\n", - "Q62018324\n", - "date_information\n", - "Q3275248\n", - "Q3670080\n", - "date_information\n", - "Q624927\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q53679499\n", - "Q55229541\n", - "Q38460115\n", - "date_information\n", - "Q20888806\n", - "date_information\n", - "Q60738525\n", - "Q48672781\n", - "Q4223893\n", - "Q24484604\n", - "Q565344\n", - "Q565344\n", - "Q7487312\n", - "date_information\n", - "date_information\n", - "Q24884893\n", - "Q24884895\n", - "Q24884895\n", - "Q24884901\n", - "Q19800856\n", - "date_information\n", - "date_information\n", - "Q4361488\n", - "Q63927162\n", - "Q75270869\n", - "date_information\n", - "Q43476870\n", - "Q43476870\n", - "Q59849898\n", - "Q56604165\n", - "Q19796666\n", - "date_information\n", - "date_information\n", - "Q55217305\n", - "Q49201649\n", - "Q1953994\n", - "Q1953994\n", - "date_information\n", - "Q27\n", - "Q60737714\n", - "date_information\n", - "date_information\n", - "Q22315057\n", - "Q48671481\n", - "date_information\n", - "date_information\n", - "Q16565369\n", - "Q60738678\n", - "date_information\n", - "Q21012765\n", - "date_information\n", - "date_information\n", - "Q55779954\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75304204\n", - "date_information\n", - "Q75272765\n", - "Q55215615\n", - "Q55215630\n", - "Q55215630\n", - "Q27\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1294270\n", - "Q1110722\n", - "Q1110722\n", - "date_information\n", - "Q774365\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56886898\n", - "Q16366420\n", - "Q75301191\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28452116\n", - "date_information\n", - "Q1677908\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75388720\n", - "date_information\n", - "Q48767646\n", - "date_information\n", - "Q17332264\n", - "date_information\n", - "date_information\n", - "Q54958519\n", - "Q24327817\n", - "Q13636928\n", - "Q65491485\n", - "Q65677532\n", - "date_information\n", - "Q75755608\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75249451\n", - "date_information\n", - "Q3731987\n", - "Q63924156\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q22119552\n", - "date_information\n", - "date_information\n", - "Q75873273\n", - "Q12929174\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q56452582\n", - "Q53451557\n", - "date_information\n", - "Q1464130\n", - "Q75404984\n", - "date_information\n", - "date_information\n", - "Q60738682\n", - "Q56562110\n", - "Q54020839\n", - "Q64140897\n", - "Q63370550\n", - "Q59967584\n", - "Q28064471\n", - "Q5495681\n", - "date_information\n", - "date_information\n", - "Q75547305\n", - "date_information\n", - "date_information\n", - "Q75549567\n", - "Q7312\n", - "Q7312\n", - "Q21857711\n", - "Q24728180\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q3823913\n", - "date_information\n", - "date_information\n", - "Q75958041\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q58824947\n", - "Q16075048\n", - "date_information\n", - "Q54868288\n", - "date_information\n", - "Q4349877\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q1013447\n", - "Q5623543\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q75278979\n", - "Q60748770\n", - "Q65042417\n", - "Q50824425\n", - "Q16647517\n", - "Q63698624\n", - "date_information\n", - "Q4587562\n", - "date_information\n", - "date_information\n", - "Q60788703\n", - "Q18234383\n", - "date_information\n", - "Q75257668\n", - "date_information\n", - "date_information\n", - "Q75274930\n", - "Q2287587\n", - "Q1853096\n", - "Q1853096\n", - "date_information\n", - "date_information\n", - "Q60738421\n", - "Q60738489\n", - "Q60738707\n", - "Q55609700\n", - "date_information\n", - "Q1147037\n", - "Q50323445\n", - "date_information\n", - "date_information\n", - "Q47034689\n", - "date_information\n", - "Q48671573\n", - "Q29329953\n", - "Q66486074\n", - "Q66486074\n", - "date_information\n", - "Q56072951\n", - "Q5623543\n", - "Q61704768\n", - "Q14455255\n", - "Q5261141\n", - "Q7312\n", - "Q7312\n", - "Q77899618\n", - "Q58411763\n", - "Q50649795\n", - "Q311536\n", - "date_information\n", - "Q30676547\n", - "Q34211\n", - "Q34211\n", - "date_information\n", - "date_information\n", - "Q50757990\n", - "date_information\n", - "Q55614737\n", - "Q30693497\n", - "Q30693497\n", - "Q51944500\n", - "date_information\n", - "Q47217028\n", - "Q20888806\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q18912661\n", - "Q75320950\n", - "date_information\n", - "Q16859764\n", - "Q16859764\n", - "date_information\n", - "Q55627536\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q57821032\n", - "date_information\n", - "date_information\n", - "Q60737698\n", - "Q75551209\n", - "date_information\n", - "Q56290550\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q17495115\n", - "date_information\n", - "date_information\n", - "Q13018966\n", - "Q63934645\n", - "date_information\n", - "Q63247471\n", - "Q17479036\n", - "Q65552650\n", - "Q75251464\n", - "Q20001576\n", - "Q31080455\n", - "date_information\n", - "date_information\n", - "Q27\n", - "date_information\n", - "Q4413321\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q16028462\n", - "date_information\n", - "Q15724107\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12054532\n", - "Q5298474\n", - "Q5298474\n", - "date_information\n", - "Q65090796\n", - "Q48672540\n", - "Q75270784\n", - "Q50650236\n", - "date_information\n", - "Q75256869\n", - "date_information\n", - "Q55635014\n", - "date_information\n", - "Q55357013\n", - "date_information\n", - "Q3337417\n", - "Q3147770\n", - "date_information\n", - "Q10405126\n", - "Q7312\n", - "Q7312\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q65085211\n", - "Q38460115\n", - "Q16029045\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q66979456\n", - "Q27\n", - "Q27\n", - "Q66023775\n", - "Q66023775\n", - "date_information\n", - "Q15044914\n", - "Q16681889\n", - "Q55762014\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q47064211\n", - "date_information\n", - "Q65089937\n", - "Q28220182\n", - "date_information\n", - "Q60738167\n", - "date_information\n", - "Q47004711\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q55081550\n", - "Q11071898\n", - "date_information\n", - "Q27\n", - "Q67381025\n", - "date_information\n", - "Q1147037\n", - "Q56816886\n", - "date_information\n", - "date_information\n", - "Q3602534\n", - "Q59733171\n", - "date_information\n", - "date_information\n", - "Q20823719\n", - "date_information\n", - "Q7312\n", - "Q7312\n", - "Q4144876\n", - "Q65089554\n", - "date_information\n", - "Q12049291\n", - "date_information\n", - "date_information\n", - "Q12864386\n", - "Q65090283\n", - "Q50378765\n", - "Q23771092\n", - "Q56063974\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q28790662\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q21008003\n", - "date_information\n", - "date_information\n", - "Q56416551\n", - "Q3661337\n", - "date_information\n", - "Q7433528\n", - "date_information\n", - "date_information\n", - "Q24055383\n", - "Q24055383\n", - "Q28365917\n", - "Q50549252\n", - "Q75394075\n", - "Q28357921\n", - "date_information\n", - "Q48965000\n", - "Q50376598\n", - "Q50376598\n", - "Q10873771\n", - "Q1050168\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q12358705\n", - "Q4971452\n", - "Q4971452\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q50821741\n", - "Q16334588\n", - "date_information\n", - "Q16679476\n", - "Q15942114\n", - "Q8857155\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q48674404\n", - "Q75607731\n", - "Q5260256\n", - "date_information\n", - "Q13017966\n", - "date_information\n", - "Q69629123\n", - "Q2082636\n", - "date_information\n", - "Q76022994\n", - "Q27\n", - "Q48740759\n", - "date_information\n", - "Q65085346\n", - "Q39072373\n", - "Q65053202\n", - "date_information\n", - "Q59725862\n", - "date_information\n", - "Q55607923\n", - "date_information\n", - "date_information\n", - "Q61046008\n", - "date_information\n", - "date_information\n", - "Q28403529\n", - "Q7433541\n", - "Q7433541\n", - "date_information\n", - "Q56886875\n", - "date_information\n", - "Q55637523\n", - "Q65053463\n", - "date_information\n", - "Q47969761\n", - "Q11151191\n", - "Q18922345\n", - "Q18922345\n", - "Q75289998\n", - "Q63934649\n", - "date_information\n", - "Q27\n", - "Q46594130\n", - "date_information\n", - "Q60749762\n", - "date_information\n", - "Q55633296\n", - "Q2849237\n", - "Q2849239\n", - "Q2849239\n", - "Q3495180\n", - "Q15878\n", - "Q4069627\n", - "Q18602858\n", - "Q55229541\n", - "date_information\n", - "Q1296004\n", - "Q56092489\n", - "date_information\n", - "date_information\n", - "Q10960429\n", - "Q64212378\n", - "Q4171683\n", - "Q27\n", - "Q75300203\n", - "Q77903030\n", - "Q64009351\n", - "Q894965\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "date_information\n", - "Q20093131\n", - "Q25563078\n", - "Q47538202\n", - "Q3404002\n", - "Q56886868\n", - "date_information\n", - "Q12201526\n", - "Q17194684\n", - "Q56887171\n", - "Q10338720\n", - "Q42279997\n", - "Q21587815\n" - ] - } - ], - "source": [ - "missing_entities = set()\n", - "missing_entity_idx = set()\n", - "for i, row in enumerate(refined_df.itertuples()):\n", - " for trip in row.graph:\n", - " if len(trip) != 3:\n", - " print(trip)\n", - " entities = trip[0], trip[2]\n", - " for entity in entities:\n", - " if entity not in entity_set:\n", - " print(entity)\n", - " missing_entities.add(entity)\n", - " missing_entity_idx.add(i)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "6851" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(missing_entity_idx)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(24441, 7)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "refined_df.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idquestion_textanswer_textis_traingraphtypegraph_size
0cdbb82ec0baf11ebab90acde48001122Who is Boraqchin (Wife Of Ögedei)'s father-in-...Genghis KhanTrue[[Q7069334, spouse, Q7519], [Q7519, father, Q7...inference2
1228546780bdd11eba7f7acde48001122What is the date of birth of the director of f...July 21, 1926True[[Q83623, director, Q309214], [Q309214, date o...compositional2
2e5150a5a0bda11eba7f7acde48001122When did the director of film Laughter In Hell...August 25, 1963True[[Q6498212, director, Q529568], [Q529568, date...compositional2
397954d9408b011ebbd84ac1f6bf848b6Do director of film Coolie No. 1 (1995 Film) a...noTrue[[Q2996544, director, Q725970], [Q64768110, di...bridge_comparison4
413cda43c09b311ebbdb0ac1f6bf848b6Are both mountains, Serre Mourene and Monte Ga...noTrue[[Q3382389, country, Q29], [Q3861576, country,...comparison2
........................
24436e6e38bc40bdd11eba7f7acde48001122Where did Coulson Wallop's father study?OxfordFalse[[Q5176173, father, Q6262835], [Q6262835, educ...compositional2
2443735fc19ce0bde11eba7f7acde48001122What is the place of birth of the director of ...IndiaFalse[[Q1637089, director, Q3417929], [Q3417929, pl...compositional2
24438523dfd220bda11eba7f7acde48001122What is the place of birth of the director of ...ReykjavíkFalse[[Q818604, director, Q1766199], [Q1766199, pla...compositional2
24439abebf78408c611ebbd90ac1f6bf848b6Were Massimo Bonanni and Gino Pasqualotto from...yesFalse[[Q460758, country of citizenship, Q38], [Q215...comparison2
24440f5175c840bdd11eba7f7acde48001122Where was the place of burial of Sultan Cem's ...Fatih MosqueFalse[[Q441311, father, Q34503], [Q34503, place of ...compositional2
\n", - "

24441 rows × 7 columns

\n", - "
" - ], - "text/plain": [ - " id \\\n", - "0 cdbb82ec0baf11ebab90acde48001122 \n", - "1 228546780bdd11eba7f7acde48001122 \n", - "2 e5150a5a0bda11eba7f7acde48001122 \n", - "3 97954d9408b011ebbd84ac1f6bf848b6 \n", - "4 13cda43c09b311ebbdb0ac1f6bf848b6 \n", - "... ... \n", - "24436 e6e38bc40bdd11eba7f7acde48001122 \n", - "24437 35fc19ce0bde11eba7f7acde48001122 \n", - "24438 523dfd220bda11eba7f7acde48001122 \n", - "24439 abebf78408c611ebbd90ac1f6bf848b6 \n", - "24440 f5175c840bdd11eba7f7acde48001122 \n", - "\n", - " question_text answer_text \\\n", - "0 Who is Boraqchin (Wife Of Ögedei)'s father-in-... Genghis Khan \n", - "1 What is the date of birth of the director of f... July 21, 1926 \n", - "2 When did the director of film Laughter In Hell... August 25, 1963 \n", - "3 Do director of film Coolie No. 1 (1995 Film) a... no \n", - "4 Are both mountains, Serre Mourene and Monte Ga... no \n", - "... ... ... \n", - "24436 Where did Coulson Wallop's father study? Oxford \n", - "24437 What is the place of birth of the director of ... India \n", - "24438 What is the place of birth of the director of ... Reykjavík \n", - "24439 Were Massimo Bonanni and Gino Pasqualotto from... yes \n", - "24440 Where was the place of burial of Sultan Cem's ... Fatih Mosque \n", - "\n", - " is_train graph \\\n", - "0 True [[Q7069334, spouse, Q7519], [Q7519, father, Q7... \n", - "1 True [[Q83623, director, Q309214], [Q309214, date o... \n", - "2 True [[Q6498212, director, Q529568], [Q529568, date... \n", - "3 True [[Q2996544, director, Q725970], [Q64768110, di... \n", - "4 True [[Q3382389, country, Q29], [Q3861576, country,... \n", - "... ... ... \n", - "24436 False [[Q5176173, father, Q6262835], [Q6262835, educ... \n", - "24437 False [[Q1637089, director, Q3417929], [Q3417929, pl... \n", - "24438 False [[Q818604, director, Q1766199], [Q1766199, pla... \n", - "24439 False [[Q460758, country of citizenship, Q38], [Q215... \n", - "24440 False [[Q441311, father, Q34503], [Q34503, place of ... \n", - "\n", - " type graph_size \n", - "0 inference 2 \n", - "1 compositional 2 \n", - "2 compositional 2 \n", - "3 bridge_comparison 4 \n", - "4 comparison 2 \n", - "... ... ... \n", - "24436 compositional 2 \n", - "24437 compositional 2 \n", - "24438 compositional 2 \n", - "24439 comparison 2 \n", - "24440 compositional 2 \n", - "\n", - "[24441 rows x 7 columns]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped.\n", - "refined_df.reset_index(inplace=True, drop=True)\n", - "refined_df" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "is_train\n", - "True 10969\n", - "False 6621\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cleaned_df = refined_df.drop(missing_entity_idx)\n", - "cleaned_df['is_train'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2024-08-07 17:32:18-- https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\n", - "Resolving www.dropbox.com (www.dropbox.com)... 162.125.13.18, 2620:100:6017:18::a27d:212\n", - "Connecting to www.dropbox.com (www.dropbox.com)|162.125.13.18|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://www.dropbox.com/scl/fi/6bjl5v1fqocokbpeui29t/wikidata5m_all_triplet.txt.gz?rlkey=wnmo016yogiz3nkl8ljk8kcbv [following]\n", - "--2024-08-07 17:32:18-- https://www.dropbox.com/scl/fi/6bjl5v1fqocokbpeui29t/wikidata5m_all_triplet.txt.gz?rlkey=wnmo016yogiz3nkl8ljk8kcbv\n", - "Reusing existing connection to www.dropbox.com:443.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline/CYN8GVBH1Zf45p2xk52RBNz9ng3i0Ft1d1TxWAf6B-X7DoxdfhtcVWojbEFgzQYhMW8HbHQQa9nCpbENjihUPdjoMFziDthGVzFdtNwtA_m4wBsxbvfNcJq3GH-vafjEMNNl-UW48NdeJrm_QsSqsTNg/file# [following]\n", - "--2024-08-07 17:32:18-- https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline/CYN8GVBH1Zf45p2xk52RBNz9ng3i0Ft1d1TxWAf6B-X7DoxdfhtcVWojbEFgzQYhMW8HbHQQa9nCpbENjihUPdjoMFziDthGVzFdtNwtA_m4wBsxbvfNcJq3GH-vafjEMNNl-UW48NdeJrm_QsSqsTNg/file\n", - "Resolving uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com (uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com)... 162.125.13.15, 2620:100:6017:15::a27d:20f\n", - "Connecting to uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com (uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com)|162.125.13.15|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: /cd/0/inline2/CYMsbtoic8kpX2pJ7dhd5-jfQA2se0Jz9bzro30yCIjWHIka93WDTuMt4Ic77oOJSQwfgWsHWSG7D9nyW2Z3q0o0kWM7G6lBWHdJCPB7cuCgdOPLtVzh30BBMccTNM_3ZdVrimhs4DHLYSVuB7daEKfazziXl3Umlhn8PY7-The6RY9SO1ZPj4qBFZjKGbJ9f27UNgwvq5YW8WjTpguwp-1MHBeMBGirO2nVCrCc7lIXaKxNCOmjDnEtWXixKWpIa4k8CWwkZUKc9Torq6dJ2_jsjC6fh7f2RgJlbFPCJaAE_23sXfGZT6FcquKeCl2gDZByvPTrZpMWbYOhceWIG_uYsBUUyMRan1OZcpmsvaabGSm6mhzDqI4n6g4ltrNQGAI/file [following]\n", - "--2024-08-07 17:32:19-- https://uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com/cd/0/inline2/CYMsbtoic8kpX2pJ7dhd5-jfQA2se0Jz9bzro30yCIjWHIka93WDTuMt4Ic77oOJSQwfgWsHWSG7D9nyW2Z3q0o0kWM7G6lBWHdJCPB7cuCgdOPLtVzh30BBMccTNM_3ZdVrimhs4DHLYSVuB7daEKfazziXl3Umlhn8PY7-The6RY9SO1ZPj4qBFZjKGbJ9f27UNgwvq5YW8WjTpguwp-1MHBeMBGirO2nVCrCc7lIXaKxNCOmjDnEtWXixKWpIa4k8CWwkZUKc9Torq6dJ2_jsjC6fh7f2RgJlbFPCJaAE_23sXfGZT6FcquKeCl2gDZByvPTrZpMWbYOhceWIG_uYsBUUyMRan1OZcpmsvaabGSm6mhzDqI4n6g4ltrNQGAI/file\n", - "Reusing existing connection to uc7f5d4e5511e66845dfd9483af4.dl.dropboxusercontent.com:443.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 177025973 (169M) [application/octet-stream]\n", - "Saving to: ‘wikidata5m_all_triplet.txt.gz’\n", - "\n", - "wikidata5m_all_trip 100%[===================>] 168.82M 96.2MB/s in 1.8s \n", - "\n", - "2024-08-07 17:32:21 (96.2 MB/s) - ‘wikidata5m_all_triplet.txt.gz’ saved [177025973/177025973]\n", - "\n" - ] - } - ], - "source": [ - "# Finally, we do the knowledge graph from wikidata5m\n", - "!wget \"https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\"" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "!gzip -d \"wikidata5m_all_triplet.txt.gz\"" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Q29387131\tP31\tQ5\n", - "\n" - ] - } - ], - "source": [ - "with open('wikidata5m_all_triplet.txt') as f:\n", - " print(f.readline())" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "# Substitute entity codes with their aliases\n", - "# Picking the first alias for each entity (rather arbitrarily)\n", - "alias_map = {}\n", - "rel_alias_map = {}\n", - "for line in open('wikidata5m_entity.txt'):\n", - " parts = line.strip().split('\\t')\n", - " entity_id = parts[0]\n", - " aliases = parts[1:]\n", - " alias_map[entity_id] = aliases[0]\n", - "for line in open('wikidata5m_relation.txt'):\n", - " parts = line.strip().split('\\t')\n", - " relation_id = parts[0]\n", - " relation_name = parts[1]\n", - " rel_alias_map[relation_id] = relation_name" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "21354359it [00:39, 543970.68it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Missing aliases: 15704/64063077\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "full_graph = []\n", - "missing_total = 0\n", - "total = 0\n", - "for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')):\n", - " src, rel, dst = line.strip().split('\\t')\n", - " if src not in alias_map:\n", - " missing_total += 1\n", - " if dst not in alias_map:\n", - " missing_total += 1\n", - " if rel not in rel_alias_map:\n", - " missing_total += 1\n", - " total += 3\n", - " full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)])\n", - "print(f\"Missing aliases: {missing_total}/{total}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['lalit kumar goel', 'instance of', 'Huamn'],\n", - " ['ugo riccarelli', 'languages spoken, written or signed', 'italian word'],\n", - " ['Road to Paradise (film)', 'director', 'beaudine, william'],\n", - " ['robert van der horst', 'country of citizenship', 'Reino Hulandes'],\n", - " ['wesley pionteck souza', 'member of sports team', 'Santos F.C.'],\n", - " ['Hezekiah Oladipo Davies', 'instance of', 'Huamn'],\n", - " ['kirakossian', 'country of citizenship', 'Hayastani Hanrapetut’yun'],\n", - " ['mountfair, virginia',\n", - " 'located in the administrative territorial entity',\n", - " 'albermarle county, virginia'],\n", - " ['juan pascual azorin', 'place of birth', 'yecla, spain'],\n", - " ['The Twilight Zone/Time Enough at Last',\n", - " 'followed by',\n", - " 'perchance to dream (twilight zone episode)']]" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "full_graph[:10]" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "torch.save(full_graph, 'wikimultihopqa_full_graph.pt')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Generating Subgraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from profiling_utils import create_remote_backend_from_triplets\n", - "from rag_feature_store import SentenceTransformerFeatureStore\n", - "from rag_graph_store import NeighborSamplingRAGGraphStore\n", - "from torch_geometric.loader import RAGQueryLoader\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", - "from torch_geometric.data import get_features_for_triplets_groups, Data" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={\"batch_size\": 256}).load()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5271ab50be1c..82cc574d73f7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -18,14 +18,12 @@ try: from pcst_fast import pcst_fast - WITH_PCST = True except ImportError: WITH_PCST = False try: import datasets - WITH_DATASETS = True except ImportError: WITH_DATASETS = False @@ -43,15 +41,10 @@ def retrieval_via_pcst( ) -> Tuple[Data, str]: c = 0.01 if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = (textual_nodes.to_csv(index=False) + - "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"])) - graph = Data( - x=graph.x, - edge_index=graph.edge_index, - edge_attr=graph.edge_attr, - num_nodes=graph.num_nodes, - ) + desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) + graph = Data(x=graph.x, edge_index=graph.edge_index, + edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) return graph, desc root = -1 # unrooted @@ -131,8 +124,8 @@ def retrieval_via_pcst( n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = (n.to_csv(index=False) + "\n" + - e.to_csv(index=False, columns=["src", "edge_attr", "dst"])) + desc = n.to_csv(index=False) + "\n" + e.to_csv( + index=False, columns=["src", "edge_attr", "dst"]) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -206,7 +199,7 @@ def download(self) -> None: torch.arange(len(dataset["validation"])) + len(dataset["train"]), "test": torch.arange(len(dataset["test"])) + len(dataset["train"]) + - len(dataset["validation"]), + len(dataset["validation"]) } def process(self) -> None: diff --git a/torch_geometric/loader/__init__.py b/torch_geometric/loader/__init__.py index 7e83c35befb6..266f498a113b 100644 --- a/torch_geometric/loader/__init__.py +++ b/torch_geometric/loader/__init__.py @@ -22,7 +22,6 @@ from .prefetch import PrefetchLoader from .cache import CachedLoader from .mixin import AffinityMixin -from .rag_loader import RAGQueryLoader __all__ = classes = [ 'DataLoader', @@ -51,7 +50,6 @@ 'PrefetchLoader', 'CachedLoader', 'AffinityMixin', - 'RAGQueryLoader', ] RandomNodeSampler = deprecated( diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py deleted file mode 100644 index f4e87f83404b..000000000000 --- a/torch_geometric/loader/rag_loader.py +++ /dev/null @@ -1,88 +0,0 @@ -from abc import abstractmethod -from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union - -from torch_geometric.data import Data, FeatureStore, HeteroData -from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput -from torch_geometric.typing import InputEdges, InputNodes - - -class RAGFeatureStore(Protocol): - """Feature store for remote GNN RAG backend.""" - @abstractmethod - def retrieve_seed_nodes(self, query: Any, **kwargs) -> InputNodes: - """Makes a comparison between the query and all the nodes to get all - the closest nodes. Return the indices of the nodes that are to be seeds - for the RAG Sampler. - """ - ... - - @abstractmethod - def retrieve_seed_edges(self, query: Any, **kwargs) -> InputEdges: - """Makes a comparison between the query and all the edges to get all - the closest nodes. Returns the edge indices that are to be the seeds - for the RAG Sampler. - """ - ... - - @abstractmethod - def load_subgraph( - self, sample: Union[SamplerOutput, HeteroSamplerOutput] - ) -> Union[Data, HeteroData]: - """Combines sampled subgraph output with features in a Data object.""" - ... - - -class RAGGraphStore(Protocol): - """Graph store for remote GNN RAG backend.""" - @abstractmethod - def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, - **kwargs) -> Union[SamplerOutput, HeteroSamplerOutput]: - """Sample a subgraph using the seeded nodes and edges.""" - ... - - @abstractmethod - def register_feature_store(self, feature_store: FeatureStore): - """Register a feature store to be used with the sampler. Samplers need - info from the feature store in order to work properly on HeteroGraphs. - """ - ... - - -# TODO: Make compatible with Heterographs - - -class RAGQueryLoader: - def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], - local_filter: Optional[Callable[[Data, Any], Data]] = None, - seed_nodes_kwargs: Optional[Dict[str, Any]] = None, - seed_edges_kwargs: Optional[Dict[str, Any]] = None, - sampler_kwargs: Optional[Dict[str, Any]] = None, - loader_kwargs: Optional[Dict[str, Any]] = None): - fstore, gstore = data - self.feature_store = fstore - self.graph_store = gstore - self.graph_store.register_feature_store(self.feature_store) - self.local_filter = local_filter - self.seed_nodes_kwargs = seed_nodes_kwargs or {} - self.seed_edges_kwargs = seed_edges_kwargs or {} - self.sampler_kwargs = sampler_kwargs or {} - self.loader_kwargs = loader_kwargs or {} - - def query(self, query: Any) -> Data: - """Retrieve a subgraph associated with the query with all its feature - attributes. - """ - seed_nodes = self.feature_store.retrieve_seed_nodes( - query, **self.seed_nodes_kwargs) - seed_edges = self.feature_store.retrieve_seed_edges( - query, **self.seed_edges_kwargs) - - subgraph_sample = self.graph_store.sample_subgraph( - seed_nodes, seed_edges, **self.sampler_kwargs) - - data = self.feature_store.load_subgraph(sample=subgraph_sample, - **self.loader_kwargs) - - if self.local_filter: - data = self.local_filter(data, query) - return data diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 0a9182bdf858..573434fc00fa 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -61,7 +61,6 @@ def __init__( num_gnn_heads: int = 4, mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096, - mlp_out_tokens: int = 1, ) -> None: super().__init__() self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) @@ -116,11 +115,9 @@ def __init__( self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), - nn.Linear(mlp_hidden_dim, mlp_out_dim*mlp_out_tokens), + nn.Linear(mlp_hidden_dim, mlp_out_dim), ).to(self.llm_device) - self.mlp_out_tokens = mlp_out_tokens - self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, node_feat, edge_index, edge_attr, batch): diff --git a/torch_geometric/profile/__init__.py b/torch_geometric/profile/__init__.py index 22d3039f4c83..833ee657d0e7 100644 --- a/torch_geometric/profile/__init__.py +++ b/torch_geometric/profile/__init__.py @@ -20,7 +20,6 @@ get_gpu_memory_from_nvidia_smi, get_model_size, ) -from .nvtx import nvtxit __all__ = [ 'profileit', @@ -39,7 +38,6 @@ 'get_gpu_memory_from_nvidia_smi', 'get_gpu_memory_from_ipex', 'benchmark', - 'nvtxit', ] classes = __all__ diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py deleted file mode 100644 index 122ddfad916f..000000000000 --- a/torch_geometric/profile/nvtx.py +++ /dev/null @@ -1,55 +0,0 @@ -from functools import wraps -from typing import Optional - -import torch - -CUDA_PROFILE_STARTED = False - - -def begin_cuda_profile(): - global CUDA_PROFILE_STARTED - prev_state = CUDA_PROFILE_STARTED - if prev_state is False: - CUDA_PROFILE_STARTED = True - torch.cuda.cudart().cudaProfilerStart() - return prev_state - - -def end_cuda_profile(prev_state: bool): - global CUDA_PROFILE_STARTED - CUDA_PROFILE_STARTED = prev_state - if prev_state is False: - torch.cuda.cudart().cudaProfilerStop() - - -def nvtxit(name: Optional[str] = None, n_warmups: int = 0, - n_iters: Optional[int] = None): - def nvtx(func): - - nonlocal name - iters_so_far = 0 - if name is None: - name = func.__name__ - - @wraps(func) - def wrapper(*args, **kwargs): - nonlocal iters_so_far - if not torch.cuda.is_available(): - return func(*args, **kwargs) - elif iters_so_far < n_warmups: - iters_so_far += 1 - return func(*args, **kwargs) - elif n_iters is None or iters_so_far < n_iters + n_warmups: - prev_state = begin_cuda_profile() - torch.cuda.nvtx.range_push(f"{name}_{iters_so_far}") - result = func(*args, **kwargs) - torch.cuda.nvtx.range_pop() - end_cuda_profile(prev_state) - iters_so_far += 1 - return result - else: - return func(*args, **kwargs) - - return wrapper - - return nvtx diff --git a/torch_geometric/profile/profiler.py b/torch_geometric/profile/profiler.py index 4b1376f54998..0ea8f73ad584 100644 --- a/torch_geometric/profile/profiler.py +++ b/torch_geometric/profile/profiler.py @@ -1,7 +1,6 @@ import functools from collections import OrderedDict, defaultdict, namedtuple from typing import Any, List, NamedTuple, Optional, Tuple -import re import torch import torch.profiler as torch_profiler @@ -303,14 +302,13 @@ def _layer_trace( layer_names = [] layer_stats = [] for format_line in format_lines: # body - # get current line's level based on number of '-' in prefix - current_line_level = len(re.match(r'^(-+).*', format_line[0]).group(1)) if format_line[1] == '': # key line - key_dict[current_line_level] = format_line[0] + key_dict[format_line[0].count("-")] = format_line[0] else: # must print # get current line's level + curr_level = format_line[0].count("-") par_str = "" - for i in range(1, current_line_level): + for i in range(1, curr_level): par_str += key_dict[i] curr_key = par_str + format_line[0] layer_names.append(curr_key) From 8c340af15b74adc22775047f57f875f1e42c6b6c Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 13:59:32 -0700 Subject: [PATCH 649/752] fix imports --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index a6af4afb3eaf..7ef79cc47b8f 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -18,7 +18,8 @@ from tqdm import tqdm from torch_geometric import seed_everything -from torch_geometric.datasets import WebQSPDataset +from torch_geometric.data import Dataset +from torch_geometric.datasets import UpdatedWebQSPDataset, WebQSPDataset from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM From 8947a9d55361cde37eb528f20cd788d6a9def586 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 14:01:57 -0700 Subject: [PATCH 650/752] lint --- examples/llm_plus_gnn/doc/media/flowchart.svg | 2 +- .../llm_plus_gnn/doc/media/multihop_example.svg | 2 +- test/data/test_large_graph_indexer.py | 5 ++--- torch_geometric/datasets/updated_web_qsp_dataset.py | 13 +++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/doc/media/flowchart.svg b/examples/llm_plus_gnn/doc/media/flowchart.svg index 8d24d4a2c309..188d37b14f41 100644 --- a/examples/llm_plus_gnn/doc/media/flowchart.svg +++ b/examples/llm_plus_gnn/doc/media/flowchart.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/examples/llm_plus_gnn/doc/media/multihop_example.svg b/examples/llm_plus_gnn/doc/media/multihop_example.svg index a6cb75681b48..42e2eaf36afa 100644 --- a/examples/llm_plus_gnn/doc/media/multihop_example.svg +++ b/examples/llm_plus_gnn/doc/media/multihop_example.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/test/data/test_large_graph_indexer.py b/test/data/test_large_graph_indexer.py index 4a5ba9ec56ee..840299ba4934 100644 --- a/test/data/test_large_graph_indexer.py +++ b/test/data/test_large_graph_indexer.py @@ -19,9 +19,8 @@ # create possible nodes and edges for graph strkeys = string.ascii_letters + string.digits -NODE_POOL = list( - set(["".join(random.sample(strkeys, 10)) for i in range(1000)])) -EDGE_POOL = list(set(["".join(random.sample(strkeys, 10)) for i in range(50)])) +NODE_POOL = list({"".join(random.sample(strkeys, 10)) for i in range(1000)}) +EDGE_POOL = list({"".join(random.sample(strkeys, 10)) for i in range(50)}) def featurize(s: str) -> int: diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 57f031a293d7..e2aa29257d52 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -237,18 +237,18 @@ def download(self) -> None: self.raw_dataset = self.raw_dataset.select(range(self.limit)) self.split_idxs = { "train": - torch.arange(self.limit//2), + torch.arange(self.limit // 2), "val": - torch.arange(self.limit//4) + self.limit//2, + torch.arange(self.limit // 4) + self.limit // 2, "test": - torch.arange(self.limit//4) + self.limit//2 + - self.limit//4, + torch.arange(self.limit // 4) + self.limit // 2 + + self.limit // 4, } self._save_raw_data() def _get_trips(self) -> Iterator[TripletLike]: return chain.from_iterable( - (iter(ds["graph"]) for ds in self.raw_dataset)) + iter(ds["graph"]) for ds in self.raw_dataset) def _build_graph(self) -> None: trips = self._get_trips() @@ -321,7 +321,8 @@ def _retrieve_subgraphs(self) -> None: def process(self) -> None: self._load_raw_data() - self.model = SentenceTransformer('sentence-transformers/all-roberta-large-v1').to(self.device) + self.model = SentenceTransformer( + 'sentence-transformers/all-roberta-large-v1').to(self.device) self.model.eval() if not os.path.exists(self.processed_paths[-1]): print("Encoding graph...") From 8b5a4622a3cec9566a225a58d8af4f728bee0575 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 16:57:46 -0700 Subject: [PATCH 651/752] mypy purgatory 1 --- torch_geometric/data/large_graph_indexer.py | 10 ++++++---- torch_geometric/datasets/updated_web_qsp_dataset.py | 12 +++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 59c2f062bbc1..e82e092ed485 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -18,6 +18,7 @@ ) import torch +from torch import Tensor from tqdm import tqdm from torch_geometric.data import Data @@ -45,11 +46,12 @@ def ordered_set(values: Iterable[Hashable]) -> List[Hashable]: EDGE_KEYS = {EDGE_PID, EDGE_HEAD, EDGE_RELATION, EDGE_TAIL, EDGE_INDEX} +FeatureValueType = Sequence[Any] | Tensor @dataclass class MappedFeature: name: str - values: Sequence[Any] + values: FeatureValueType def __eq__(self, value: "MappedFeature") -> bool: eq = self.name == value.name @@ -180,7 +182,7 @@ def get_unique_node_features( def add_node_feature( self, new_feature_name: str, - new_feature_vals: Sequence[Any], + new_feature_vals: FeatureValueType, map_from_feature: str = NODE_PID, ) -> None: @@ -270,7 +272,7 @@ def get_unique_edge_features( def add_edge_feature( self, new_feature_name: str, - new_feature_vals: Sequence[Any], + new_feature_vals: FeatureValueType, map_from_feature: str = EDGE_PID, ) -> None: @@ -297,7 +299,7 @@ def get_edge_features( self, feature_name: str = EDGE_PID, pids: Optional[Iterable[Hashable]] = None, - ) -> Sequence[Any]: + ) -> List[Any]: if feature_name in self._mapped_edge_features: values = self.edge_attr[feature_name].values else: diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index e2aa29257d52..12bf96550258 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -160,7 +160,7 @@ def retrieval_via_pcst( return data, desc -def preprocess_triplet(triplet: TripletLike) -> TripletLike: +def preprocess_triplet(triplet: Tuple[str, str, str]) -> TripletLike: h, r, t = triplet return h.lower(), r, t.lower() @@ -256,17 +256,17 @@ def _build_graph(self) -> None: trips, pre_transform=preprocess_triplet) # Nodes: - nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256) + nodes: List[str] = self.indexer.get_unique_node_features() + x = self.model.encode(nodes, batch_size=256) # type: ignore self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: - edges = self.indexer.get_unique_edge_features( + edges: List[str] = self.indexer.get_unique_edge_features( feature_name=EDGE_RELATION) edge_attr = self.model.encode(edges, batch_size=256) self.indexer.add_edge_feature( new_feature_name="edge_attr", - new_feature_vals=edge_attr, + new_feature_vals=edge_attr, # type: ignore map_from_feature=EDGE_RELATION, ) @@ -281,8 +281,6 @@ def _retrieve_subgraphs(self) -> None: print("Retrieving subgraphs...") textual_nodes = self.textual_nodes textual_edges = self.textual_edges - graph = None - graph_gen = None if self.whole_graph_retrieval: graph = self.indexer.to_data(node_feature_name="x", edge_feature_name="edge_attr") From d4c20c478af3b910cf84b84e937070c0f756401b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 17:05:53 -0700 Subject: [PATCH 652/752] mypy purgatory 2 --- torch_geometric/data/large_graph_indexer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index e82e092ed485..88723bc54403 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -15,6 +15,7 @@ Sequence, Set, Tuple, + Union, ) import torch @@ -46,7 +47,8 @@ def ordered_set(values: Iterable[Hashable]) -> List[Hashable]: EDGE_KEYS = {EDGE_PID, EDGE_HEAD, EDGE_RELATION, EDGE_TAIL, EDGE_INDEX} -FeatureValueType = Sequence[Any] | Tensor +FeatureValueType = Union[Sequence[Any], Tensor] + @dataclass class MappedFeature: From d0c6dfab7d5cd2160cdaf4a7bb8eabe4827022f6 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 17:19:11 -0700 Subject: [PATCH 653/752] mypy purgatory 3 --- torch_geometric/datasets/updated_web_qsp_dataset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index 12bf96550258..eebde35998a3 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -160,9 +160,9 @@ def retrieval_via_pcst( return data, desc -def preprocess_triplet(triplet: Tuple[str, str, str]) -> TripletLike: +def preprocess_triplet(triplet: TripletLike) -> TripletLike: h, r, t = triplet - return h.lower(), r, t.lower() + return str(h).lower(), str(r), str(t).lower() class UpdatedWebQSPDataset(InMemoryDataset): @@ -256,14 +256,14 @@ def _build_graph(self) -> None: trips, pre_transform=preprocess_triplet) # Nodes: - nodes: List[str] = self.indexer.get_unique_node_features() + nodes = self.indexer.get_unique_node_features() x = self.model.encode(nodes, batch_size=256) # type: ignore self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: - edges: List[str] = self.indexer.get_unique_edge_features( + edges = self.indexer.get_unique_edge_features( feature_name=EDGE_RELATION) - edge_attr = self.model.encode(edges, batch_size=256) + edge_attr = self.model.encode(edges, batch_size=256) # type: ignore self.indexer.add_edge_feature( new_feature_name="edge_attr", new_feature_vals=edge_attr, # type: ignore @@ -275,7 +275,7 @@ def _build_graph(self) -> None: def _retrieve_subgraphs(self) -> None: print("Encoding questions...") - self.questions = [ds["question"] for ds in self.raw_dataset] + self.questions = [str(ds["question"]) for ds in self.raw_dataset] q_embs = self.model.encode(self.questions, batch_size=256) list_of_graphs = [] print("Retrieving subgraphs...") From b835caa1e4a4051ac2c399a2a03974f2c304f73a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 17:23:50 -0700 Subject: [PATCH 654/752] Changelog 0 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34677ae476f5..704c916c1490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added `data.LargeGraphIndexer` ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `nn.models.GRetriever` ([#9167](https://github.com/pyg-team/pytorch_geometric/pull/9167)) - Added the `WebQSPDataset` dataset ([#9481](https://github.com/pyg-team/pytorch_geometric/pull/9481)) - Added the `nn.models.GRetriever` model ([#9480](https://github.com/pyg-team/pytorch_geometric/pull/9480)) From 79d70c347f518afd1d54b739d22ea5eb345cd11c Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 17:25:21 -0700 Subject: [PATCH 655/752] mypy purgatory 4 --- torch_geometric/datasets/updated_web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index eebde35998a3..a5c9b1c9077b 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -266,7 +266,7 @@ def _build_graph(self) -> None: edge_attr = self.model.encode(edges, batch_size=256) # type: ignore self.indexer.add_edge_feature( new_feature_name="edge_attr", - new_feature_vals=edge_attr, # type: ignore + new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION, ) From a15b141d49dd5dfda3a2511a3bf3f9ca073eeab6 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 03:11:28 -0700 Subject: [PATCH 656/752] docstrings --- torch_geometric/data/large_graph_indexer.py | 162 +++++++++++++++++- .../datasets/updated_web_qsp_dataset.py | 13 ++ 2 files changed, 170 insertions(+), 5 deletions(-) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 88723bc54403..4e180ee5ec25 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -24,8 +24,6 @@ from torch_geometric.data import Data -# Is there a multiprocessing-friendly implementation of this? - TripletLike = Tuple[Hashable, Hashable, Hashable] @@ -76,6 +74,19 @@ def __init__( node_attr: Optional[Dict[str, List[Any]]] = None, edge_attr: Optional[Dict[str, List[Any]]] = None, ) -> None: + r"""Constructs a new index that uniquely catalogs each node and edge + by id. Not meant to be used directly. + + Args: + nodes (Iterable[Hashable]): Node ids in the graph. + edges (Iterable[TripletLike]): Edge ids in the graph. + node_attr (Optional[Dict[str, List[Any]]], optional): Mapping node + attribute name and list of their values in order of unique node + ids. Defaults to None. + edge_attr (Optional[Dict[str, List[Any]]], optional): Mapping edge + attribute name and list of their values in order of unique edge + ids. Defaults to None. + """ self._nodes: Dict[Hashable, int] = dict() self._edges: Dict[TripletLike, int] = dict() @@ -139,6 +150,20 @@ def from_triplets( triplets: Iterable[TripletLike], pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> "LargeGraphIndexer": + r"""Generate a new index from a series of triplets that represent edge + relations between nodes. + Formatted like (source_node, edge, dest_node). + + Args: + triplets (Iterable[TripletLike]): Series of triplets representing + knowledge graph relations. + pre_transform (Optional[Callable[[TripletLike], TripletLike]]): + Optional preprocessing function to apply to triplets. + Defaults to None. + + Returns: + LargeGraphIndexer: Index of unique nodes and edges. + """ # NOTE: Right now assumes that all trips can be loaded into memory nodes = set() edges = set() @@ -165,12 +190,32 @@ def apply_transform( @classmethod def collate(cls, graphs: Iterable["LargeGraphIndexer"]) -> "LargeGraphIndexer": + r"""Combines a series of large graph indexes into a single large graph + index. + + Args: + graphs (Iterable["LargeGraphIndexer"]): Indices to be + combined. + + Returns: + LargeGraphIndexer: Singular unique index for all nodes and edges + in input indices. + """ # FIXME Needs to merge node attrs and edge attrs? trips = chain.from_iterable([graph.to_triplets() for graph in graphs]) return cls.from_triplets(trips) def get_unique_node_features( self, feature_name: str = NODE_PID) -> List[Hashable]: + r"""Get all the unique values for a specific node attribute. + + Args: + feature_name (str, optional): Name of feature to get. + Defaults to NODE_PID. + + Returns: + List[Hashable]: List of unique values for the specified feature. + """ try: if feature_name in self._mapped_node_features: raise IndexError( @@ -187,7 +232,17 @@ def add_node_feature( new_feature_vals: FeatureValueType, map_from_feature: str = NODE_PID, ) -> None: - + r"""Adds a new feature that corresponds to each unique node in + the graph. + + Args: + new_feature_name (str): Name to call the new feature. + new_feature_vals (FeatureValueType): Values to map for that + new feature. + map_from_feature (str, optional): Key of feature to map from. + Size must match the number of feature values. + Defaults to NODE_PID. + """ if new_feature_name in self.node_attr: raise AttributeError("Features cannot be overridden once created") if map_from_feature in self._mapped_node_features: @@ -212,6 +267,18 @@ def get_node_features( feature_name: str = NODE_PID, pids: Optional[Iterable[Hashable]] = None, ) -> List[Any]: + r"""Get node feature values for a given set of unique node ids. + Returned values are not necessarily unique. + + Args: + feature_name (str, optional): Name of feature to fetch. Defaults + to NODE_PID. + pids (Optional[Iterable[Hashable]], optional): Node ids to fetch + for. Defaults to None, which fetches all nodes. + + Returns: + List[Any]: Node features corresponding to the specified ids. + """ if feature_name in self._mapped_node_features: values = self.node_attr[feature_name].values else: @@ -231,6 +298,9 @@ def get_node_features_iter( pids: Optional[Iterable[Hashable]] = None, index_only: bool = False, ) -> Iterator[Any]: + """Iterator version of get_node_features. If index_only is True, + yields indices instead of values. + """ if pids is None: pids = self.node_attr[NODE_PID] @@ -262,6 +332,15 @@ def get_node_features_iter( def get_unique_edge_features( self, feature_name: str = EDGE_PID) -> List[Hashable]: + r"""Get all the unique values for a specific edge attribute. + + Args: + feature_name (str, optional): Name of feature to get. + Defaults to EDGE_PID. + + Returns: + List[Hashable]: List of unique values for the specified feature. + """ try: if feature_name in self._mapped_edge_features: raise IndexError( @@ -277,7 +356,17 @@ def add_edge_feature( new_feature_vals: FeatureValueType, map_from_feature: str = EDGE_PID, ) -> None: - + r"""Adds a new feature that corresponds to each unique edge in + the graph. + + Args: + new_feature_name (str): Name to call the new feature. + new_feature_vals (FeatureValueType): Values to map for that new + feature. + map_from_feature (str, optional): Key of feature to map from. + Size must match the number of feature values. + Defaults to EDGE_PID. + """ if new_feature_name in self.edge_attr: raise AttributeError("Features cannot be overridden once created") if map_from_feature in self._mapped_edge_features: @@ -302,6 +391,18 @@ def get_edge_features( feature_name: str = EDGE_PID, pids: Optional[Iterable[Hashable]] = None, ) -> List[Any]: + r"""Get edge feature values for a given set of unique edge ids. + Returned values are not necessarily unique. + + Args: + feature_name (str, optional): Name of feature to fetch. + Defaults to EDGE_PID. + pids (Optional[Iterable[Hashable]], optional): Edge ids to fetch + for. Defaults to None, which fetches all edges. + + Returns: + List[Any]: Node features corresponding to the specified ids. + """ if feature_name in self._mapped_edge_features: values = self.edge_attr[feature_name].values else: @@ -321,6 +422,9 @@ def get_edge_features_iter( pids: Optional[Iterable[TripletLike]] = None, index_only: bool = False, ) -> Iterator[Any]: + """Iterator version of get_edge_features. If index_only is True, + yields indices instead of values. + """ if pids is None: pids = self.edge_attr[EDGE_PID] @@ -429,6 +533,18 @@ def __eq__(self, value: "LargeGraphIndexer") -> bool: def to_data(self, node_feature_name: str, edge_feature_name: Optional[str] = None) -> Data: + """Return a Data object containing all the specified node and + edge features and the graph. + + Args: + node_feature_name (str): Feature to use for nodes + edge_feature_name (Optional[str], optional): Feature to use for + edges. Defaults to None. + + Returns: + Data: Data object containing the specified node and + edge features and the graph. + """ x = torch.Tensor(self.get_node_features(node_feature_name)) node_id = torch.LongTensor(range(len(x))) @@ -449,6 +565,26 @@ def get_features_for_triplets_groups( node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None ) -> Iterator[Data]: + """Given an indexer and a series of triplet groups (like a dataset), + retrieve the specified node and edge features for each triplet from the + index. + + Args: + indexer (LargeGraphIndexer): Indexer containing desired features + triplet_groups (Iterable[Iterable[TripletLike]]): List of lists of + triplets to fetch features for + node_feature_name (str, optional): Node feature to fetch. + Defaults to "x". + edge_feature_name (str, optional): edge feature to fetch. + Defaults to "edge_attr". + pre_transform (Optional[Callable[[TripletLike], TripletLike]]): + Optional preprocessing to perform on triplets. + Defaults to None. + + Yields: + Iterator[Data]: For each triplet group, yield a data object containing + the unique graph and features from the index. + """ if pre_transform is not None: def apply_transform(trips): @@ -505,7 +641,23 @@ def get_features_for_triplets( edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> Data: - + """For a given set of triplets retrieve a Data object containing the + unique graph and features from the index. + + Args: + indexer (LargeGraphIndexer): Indexer containing desired features + triplets (Iterable[TripletLike]): Triplets to fetch features for + node_feature_name (str, optional): Feature to use for node features. + Defaults to "x". + edge_feature_name (str, optional): Feature to use for edge features. + Defaults to "edge_attr". + pre_transform (Optional[Callable[[TripletLike], TripletLike]]): + Optional preprocessing function for triplets. Defaults to None. + + Returns: + Data: Data object containing the unique graph and features from the + index for the given triplets. + """ gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, edge_feature_name, pre_transform) diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py index a5c9b1c9077b..fac429d845d6 100644 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ b/torch_geometric/datasets/updated_web_qsp_dataset.py @@ -174,6 +174,19 @@ def __init__( limit: int = -1, override: bool = False, ) -> None: + """Construct a WebQSPDataset. + + Args: + root (str, optional): Path to save dataset to. Defaults to "". + force_reload (bool, optional): Whether to rerun process step. + Defaults to False. + whole_graph_retrieval (bool, optional): Whether to retrieve the + entire knowledge graph at once. Defaults to False. + limit (int, optional): Construct only the first n samples. + Defaults to -1 to construct all samples. + override (bool, optional): Whether to skip WebQSP step. Defaults + to False. + """ self.limit = limit self.whole_graph_retrieval = whole_graph_retrieval self.override = override From cc3543d82bd4cf15ebb6100632e9c9238a377222 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 14:56:37 -0700 Subject: [PATCH 657/752] unittest for updated qsp --- test/datasets/test_web_qsp_dataset.py | 34 ++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 4079f09b6484..a9c1755f1afb 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,4 +1,8 @@ -from torch_geometric.datasets import WebQSPDataset +import networkx as nx +import torch + +from torch_geometric.data import Data +from torch_geometric.datasets import UpdatedWebQSPDataset, WebQSPDataset from torch_geometric.testing import onlyFullTest, onlyOnline @@ -8,3 +12,31 @@ def test_web_qsp_dataset(): dataset = WebQSPDataset() assert len(dataset) == 4700 assert str(dataset) == "WebQSPDataset(4700)" + + +@onlyOnline +@onlyFullTest +def test_updated_web_qsp_dataset(): + def results_are_close_enough(ground_truth: Data, new_method: Data, + thresh=.8): + def _sorted_tensors_are_close(tensor1, tensor2): + return torch.all( + torch.isclose(tensor1.sort(dim=0)[0], + tensor2.sort(dim=0)[0]).float().mean( + axis=1) > thresh) + + def _graphs_are_same(tensor1, tensor2): + return nx.weisfeiler_lehman_graph_hash(nx.Graph( + tensor1.T)) == nx.weisfeiler_lehman_graph_hash( + nx.Graph(tensor2.T)) + return _sorted_tensors_are_close(ground_truth.x, new_method.x) \ + and _sorted_tensors_are_close(ground_truth.edge_attr, + new_method.edge_attr) \ + and _graphs_are_same(ground_truth.edge_index, + new_method.edge_index) + + dataset_original = WebQSPDataset() + dataset_updated = UpdatedWebQSPDataset('updated') + + for ground_truth, updated_graph in zip(dataset_original, dataset_updated): + assert results_are_close_enough(ground_truth, updated_graph) From ea3c0befcf49a09573669146e29749b468b84806 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 27 Aug 2024 03:08:37 -0700 Subject: [PATCH 658/752] address pr feedback --- torch_geometric/data/__init__.py | 12 +++- torch_geometric/data/large_graph_indexer.py | 68 +++++++++++---------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/torch_geometric/data/__init__.py b/torch_geometric/data/__init__.py index 5acf82c22467..fee215b1a357 100644 --- a/torch_geometric/data/__init__.py +++ b/torch_geometric/data/__init__.py @@ -46,9 +46,15 @@ ] helper_functions = [ - 'makedirs', 'download_url', 'download_google_url', 'extract_tar', - 'extract_zip', 'extract_bz2', 'extract_gz', 'get_features_for_triplets', - "get_features_for_triplets_groups" + 'makedirs', + 'download_url', + 'download_google_url', + 'extract_tar', + 'extract_zip', + 'extract_bz2', + 'extract_gz', + 'get_features_for_triplets', + "get_features_for_triplets_groups", ] __all__ = data_classes + remote_backend_classes + helper_functions diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 4e180ee5ec25..e1f7b594ca9f 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -26,6 +26,8 @@ TripletLike = Tuple[Hashable, Hashable, Hashable] +KnowledgeGraphLike = Iterable[TripletLike] + def ordered_set(values: Iterable[Hashable]) -> List[Hashable]: return list(dict.fromkeys(values)) @@ -70,7 +72,7 @@ class LargeGraphIndexer: def __init__( self, nodes: Iterable[Hashable], - edges: Iterable[TripletLike], + edges: KnowledgeGraphLike, node_attr: Optional[Dict[str, List[Any]]] = None, edge_attr: Optional[Dict[str, List[Any]]] = None, ) -> None: @@ -79,7 +81,7 @@ def __init__( Args: nodes (Iterable[Hashable]): Node ids in the graph. - edges (Iterable[TripletLike]): Edge ids in the graph. + edges (KnowledgeGraphLike): Edge ids in the graph. node_attr (Optional[Dict[str, List[Any]]], optional): Mapping node attribute name and list of their values in order of unique node ids. Defaults to None. @@ -147,7 +149,7 @@ def __init__( @classmethod def from_triplets( cls, - triplets: Iterable[TripletLike], + triplets: KnowledgeGraphLike, pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, ) -> "LargeGraphIndexer": r"""Generate a new index from a series of triplets that represent edge @@ -155,7 +157,7 @@ def from_triplets( Formatted like (source_node, edge, dest_node). Args: - triplets (Iterable[TripletLike]): Series of triplets representing + triplets (KnowledgeGraphLike): Series of triplets representing knowledge graph relations. pre_transform (Optional[Callable[[TripletLike], TripletLike]]): Optional preprocessing function to apply to triplets. @@ -171,7 +173,7 @@ def from_triplets( if pre_transform is not None: def apply_transform( - trips: Iterable[TripletLike]) -> Iterator[TripletLike]: + trips: KnowledgeGraphLike) -> Iterator[TripletLike]: for trip in trips: yield pre_transform(trip) @@ -419,7 +421,7 @@ def get_edge_features( def get_edge_features_iter( self, feature_name: str = EDGE_PID, - pids: Optional[Iterable[TripletLike]] = None, + pids: Optional[KnowledgeGraphLike] = None, index_only: bool = False, ) -> Iterator[Any]: """Iterator version of get_edge_features. If index_only is True, @@ -508,29 +510,6 @@ def from_disk(cls, path: str) -> "LargeGraphIndexer": return indexer - def __eq__(self, value: "LargeGraphIndexer") -> bool: - eq = True - eq &= self._nodes == value._nodes - eq &= self._edges == value._edges - eq &= self.node_attr.keys() == value.node_attr.keys() - eq &= self.edge_attr.keys() == value.edge_attr.keys() - eq &= self._mapped_node_features == value._mapped_node_features - eq &= self._mapped_edge_features == value._mapped_edge_features - - for k in self.node_attr: - eq &= isinstance(self.node_attr[k], type(value.node_attr[k])) - if isinstance(self.node_attr[k], torch.Tensor): - eq &= torch.equal(self.node_attr[k], value.node_attr[k]) - else: - eq &= self.node_attr[k] == value.node_attr[k] - for k in self.edge_attr: - eq &= isinstance(self.edge_attr[k], type(value.edge_attr[k])) - if isinstance(self.edge_attr[k], torch.Tensor): - eq &= torch.equal(self.edge_attr[k], value.edge_attr[k]) - else: - eq &= self.edge_attr[k] == value.edge_attr[k] - return eq - def to_data(self, node_feature_name: str, edge_feature_name: Optional[str] = None) -> Data: """Return a Data object containing all the specified node and @@ -558,10 +537,33 @@ def to_data(self, node_feature_name: str, return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, edge_id=edge_id, node_id=node_id) + def __eq__(self, value: "LargeGraphIndexer") -> bool: + eq = True + eq &= self._nodes == value._nodes + eq &= self._edges == value._edges + eq &= self.node_attr.keys() == value.node_attr.keys() + eq &= self.edge_attr.keys() == value.edge_attr.keys() + eq &= self._mapped_node_features == value._mapped_node_features + eq &= self._mapped_edge_features == value._mapped_edge_features + + for k in self.node_attr: + eq &= isinstance(self.node_attr[k], type(value.node_attr[k])) + if isinstance(self.node_attr[k], torch.Tensor): + eq &= torch.equal(self.node_attr[k], value.node_attr[k]) + else: + eq &= self.node_attr[k] == value.node_attr[k] + for k in self.edge_attr: + eq &= isinstance(self.edge_attr[k], type(value.edge_attr[k])) + if isinstance(self.edge_attr[k], torch.Tensor): + eq &= torch.equal(self.edge_attr[k], value.edge_attr[k]) + else: + eq &= self.edge_attr[k] == value.edge_attr[k] + return eq + def get_features_for_triplets_groups( indexer: LargeGraphIndexer, - triplet_groups: Iterable[Iterable[TripletLike]], + triplet_groups: Iterable[KnowledgeGraphLike], node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None ) -> Iterator[Data]: @@ -571,7 +573,7 @@ def get_features_for_triplets_groups( Args: indexer (LargeGraphIndexer): Indexer containing desired features - triplet_groups (Iterable[Iterable[TripletLike]]): List of lists of + triplet_groups (Iterable[KnowledgeGraphLike]): List of lists of triplets to fetch features for node_feature_name (str, optional): Node feature to fetch. Defaults to "x". @@ -636,7 +638,7 @@ def apply_transform(trips): def get_features_for_triplets( indexer: LargeGraphIndexer, - triplets: Iterable[TripletLike], + triplets: KnowledgeGraphLike, node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, @@ -646,7 +648,7 @@ def get_features_for_triplets( Args: indexer (LargeGraphIndexer): Indexer containing desired features - triplets (Iterable[TripletLike]): Triplets to fetch features for + triplets (KnowledgeGraphLike): Triplets to fetch features for node_feature_name (str, optional): Feature to use for node features. Defaults to "x". edge_feature_name (str, optional): Feature to use for edge features. From eac09a826a324ede0392832e883c20ff97307eae Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:26:23 -0700 Subject: [PATCH 659/752] Add back in all Remote backend deltas --- examples/llm_plus_gnn/profiling_utils.py | 193 ++++++++++++++++++ examples/llm_plus_gnn/rag_feature_store.py | 93 +++++++++ examples/llm_plus_gnn/rag_generate.py | 75 +++++++ .../llm_plus_gnn/rag_generate_multihop.py | 38 ++++ examples/llm_plus_gnn/rag_graph_store.py | 92 +++++++++ torch_geometric/loader/__init__.py | 2 + torch_geometric/loader/rag_loader.py | 88 ++++++++ 7 files changed, 581 insertions(+) create mode 100644 examples/llm_plus_gnn/profiling_utils.py create mode 100644 examples/llm_plus_gnn/rag_feature_store.py create mode 100644 examples/llm_plus_gnn/rag_generate.py create mode 100644 examples/llm_plus_gnn/rag_generate_multihop.py create mode 100644 examples/llm_plus_gnn/rag_graph_store.py create mode 100644 torch_geometric/loader/rag_loader.py diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py new file mode 100644 index 000000000000..2bc537f08d4a --- /dev/null +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -0,0 +1,193 @@ +from dataclasses import dataclass +from enum import Enum, auto +from typing import ( + Any, + Callable, + Dict, + Iterable, + Optional, + Protocol, + Tuple, + Type, + runtime_checkable, +) + +import torch +from torch import Tensor +from torch.nn import Module + +from torch_geometric.data import ( + FeatureStore, + GraphStore, + LargeGraphIndexer, + TripletLike, +) +from torch_geometric.data.large_graph_indexer import EDGE_RELATION +from torch_geometric.distributed import ( + LocalFeatureStore, + LocalGraphStore, + Partitioner, +) +from torch_geometric.typing import EdgeType, NodeType + +RemoteGraphBackend = Tuple[FeatureStore, GraphStore] + +# TODO: Make everything compatible with Hetero graphs aswell + + +# Adapted from LocalGraphStore +@runtime_checkable +class ConvertableGraphStore(Protocol): + @classmethod + def from_data( + cls, + edge_id: Tensor, + edge_index: Tensor, + num_nodes: int, + is_sorted: bool = False, + ) -> GraphStore: + ... + + @classmethod + def from_hetero_data( + cls, + edge_id_dict: Dict[EdgeType, Tensor], + edge_index_dict: Dict[EdgeType, Tensor], + num_nodes_dict: Dict[NodeType, int], + is_sorted: bool = False, + ) -> GraphStore: + ... + + @classmethod + def from_partition(cls, root: str, pid: int) -> GraphStore: + ... + + +# Adapted from LocalFeatureStore +@runtime_checkable +class ConvertableFeatureStore(Protocol): + @classmethod + def from_data( + cls, + node_id: Tensor, + x: Optional[Tensor] = None, + y: Optional[Tensor] = None, + edge_id: Optional[Tensor] = None, + edge_attr: Optional[Tensor] = None, + ) -> FeatureStore: + ... + + @classmethod + def from_hetero_data( + cls, + node_id_dict: Dict[NodeType, Tensor], + x_dict: Optional[Dict[NodeType, Tensor]] = None, + y_dict: Optional[Dict[NodeType, Tensor]] = None, + edge_id_dict: Optional[Dict[EdgeType, Tensor]] = None, + edge_attr_dict: Optional[Dict[EdgeType, Tensor]] = None, + ) -> FeatureStore: + ... + + @classmethod + def from_partition(cls, root: str, pid: int) -> FeatureStore: + ... + + +class RemoteDataType(Enum): + DATA = auto() + PARTITION = auto() + + +@dataclass +class RemoteGraphBackendLoader: + path: str + datatype: RemoteDataType + graph_store_type: Type[ConvertableGraphStore] + feature_store_type: Type[ConvertableFeatureStore] + + def load(self, pid: Optional[int] = None) -> RemoteGraphBackend: + if self.datatype == RemoteDataType.DATA: + data_obj = torch.load(self.path) + graph_store = self.graph_store_type.from_data( + edge_id=data_obj['edge_id'], edge_index=data_obj.edge_index, + num_nodes=data_obj.num_nodes) + feature_store = self.feature_store_type.from_data( + node_id=data_obj['node_id'], x=data_obj.x, + edge_id=data_obj['edge_id'], edge_attr=data_obj.edge_attr) + elif self.datatype == RemoteDataType.PARTITION: + if pid is None: + assert pid is not None, \ + "Partition ID must be defined for loading from a " \ + + "partitioned store." + graph_store = self.graph_store_type.from_partition(self.path, pid) + feature_store = self.feature_store_type.from_partition( + self.path, pid) + else: + raise NotImplementedError + return (feature_store, graph_store) + + +# TODO: make profilable +def create_remote_backend_from_triplets( + triplets: Iterable[TripletLike], node_embedding_model: Module, + edge_embedding_model: Module | None = None, + graph_db: Type[ConvertableGraphStore] = LocalGraphStore, + feature_db: Type[ConvertableFeatureStore] = LocalFeatureStore, + node_method_to_call: str = "forward", + edge_method_to_call: str | None = None, + pre_transform: Callable[[TripletLike], TripletLike] | None = None, + path: str = '', n_parts: int = 1, + node_method_kwargs: Optional[Dict[str, Any]] = None, + edge_method_kwargs: Optional[Dict[str, Any]] = None +) -> RemoteGraphBackendLoader: + + # Will return attribute errors for missing attributes + if not issubclass(graph_db, ConvertableGraphStore): + getattr(graph_db, "from_data") + getattr(graph_db, "from_hetero_data") + getattr(graph_db, "from_partition") + elif not issubclass(feature_db, ConvertableFeatureStore): + getattr(feature_db, "from_data") + getattr(feature_db, "from_hetero_data") + getattr(feature_db, "from_partition") + + # Resolve callable methods + node_method_kwargs = node_method_kwargs \ + if node_method_kwargs is not None else dict() + + edge_embedding_model = edge_embedding_model \ + if edge_embedding_model is not None else node_embedding_model + edge_method_to_call = edge_method_to_call \ + if edge_method_to_call is not None else node_method_to_call + edge_method_kwargs = edge_method_kwargs \ + if edge_method_kwargs is not None else node_method_kwargs + + # These will return AttributeErrors if they don't exist + node_model = getattr(node_embedding_model, node_method_to_call) + edge_model = getattr(edge_embedding_model, edge_method_to_call) + + indexer = LargeGraphIndexer.from_triplets(triplets, + pre_transform=pre_transform) + + node_feats = node_model(indexer.get_node_features(), **node_method_kwargs) + indexer.add_node_feature('x', node_feats) + + edge_feats = edge_model( + indexer.get_unique_edge_features(feature_name=EDGE_RELATION), + **edge_method_kwargs) + indexer.add_edge_feature(new_feature_name="edge_attr", + new_feature_vals=edge_feats, + map_from_feature=EDGE_RELATION) + + data = indexer.to_data(node_feature_name='x', + edge_feature_name='edge_attr') + + if n_parts == 1: + torch.save(data, path) + return RemoteGraphBackendLoader(path, RemoteDataType.DATA, graph_db, + feature_db) + else: + partitioner = Partitioner(data=data, num_parts=n_parts, root=path) + partitioner.generate_partition() + return RemoteGraphBackendLoader(path, RemoteDataType.PARTITION, + graph_db, feature_db) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py new file mode 100644 index 000000000000..ca3a0ba646d4 --- /dev/null +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -0,0 +1,93 @@ +from collections.abc import Iterable, Iterator +from typing import Any, Dict, Optional, Type, Union + +import torch +from torch import Tensor +from torch.nn import Module +from torchmetrics.functional import pairwise_cosine_similarity + +from torch_geometric.data import Data, HeteroData +from torch_geometric.distributed import LocalFeatureStore +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput +from torch_geometric.typing import InputEdges, InputNodes + + +# NOTE: Only compatible with Homogeneous graphs for now +class KNNRAGFeatureStore(LocalFeatureStore): + def __init__(self, enc_model: Type[Module], + model_kwargs: Optional[Dict[str, + Any]] = None, *args, **kwargs): + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + self.enc_model = enc_model(*args, **kwargs).to(self.device) + self.enc_model.eval() + self.model_kwargs = \ + model_kwargs if model_kwargs is not None else dict() + super().__init__() + + @property + def x(self) -> Tensor: + return self.get_tensor(group_name=None, attr_name='x') + + @property + def edge_attr(self) -> Tensor: + return self.get_tensor(group_name=(None, None), attr_name='edge_attr') + + def retrieve_seed_nodes(self, query: Any, k_nodes: int = 5) -> InputNodes: + return next(self._retrieve_seed_nodes_batch([query], k_nodes)) + + def _retrieve_seed_nodes_batch(self, query: Iterable[Any], + k_nodes: int) -> Iterator[InputNodes]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError + + query_enc = self.enc_model.encode(query, + **self.model_kwargs).to(self.device) + prizes = pairwise_cosine_similarity(query_enc, self.x.to(self.device)) + topk = min(k_nodes, len(self.x)) + for q in prizes: + _, indices = torch.topk(q, topk, largest=True) + yield indices + + def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: + return next(self._retrieve_seed_edges_batch([query], k_edges)) + + def _retrieve_seed_edges_batch(self, query: Iterable[Any], + k_edges: int) -> Iterator[InputEdges]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError + + query_enc = self.enc_model.encode(query, + **self.model_kwargs).to(self.device) + + prizes = pairwise_cosine_similarity(query_enc, + self.edge_attr.to(self.device)) + topk = min(k_edges, len(self.edge_attr)) + for q in prizes: + _, indices = torch.topk(q, topk, largest=True) + yield indices + + def load_subgraph( + self, sample: Union[SamplerOutput, HeteroSamplerOutput] + ) -> Union[Data, HeteroData]: + + if isinstance(sample, HeteroSamplerOutput): + raise NotImplementedError + + # NOTE: torch_geometric.loader.utils.filter_custom_store can be used + # here if it supported edge features + node_id = sample.node + edge_id = sample.edge + edge_index = torch.stack((sample.row, sample.col), dim=0) + x = self.x[node_id] + edge_attr = self.edge_attr[edge_id] + + return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, + node_idx=node_id, edge_idx=edge_id) + + +class SentenceTransformerFeatureStore(KNNRAGFeatureStore): + def __init__(self, *args, **kwargs): + kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') + super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py new file mode 100644 index 000000000000..d4bb628dec56 --- /dev/null +++ b/examples/llm_plus_gnn/rag_generate.py @@ -0,0 +1,75 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import RAGQueryLoader +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd + +# %% +ds = UpdatedWebQSPDataset("small_dataset", force_reload=True, limit=20) + +# %% +triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) + +# %% +questions = ds.raw_dataset['question'] +questions + +# %% +ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +num_edges = len(ds.indexer._edges) + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}) + +# %% +# Accuracy Metrics to be added to Profiler +def _eidx_helper(subg: Data, ground_truth: Data): + subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx + if isinstance(subg_eidx, torch.Tensor): + subg_eidx = subg_eidx.tolist() + if isinstance(gt_eidx, torch.Tensor): + gt_eidx = gt_eidx.tolist() + subg_e = set(subg_eidx) + gt_e = set(gt_eidx) + return subg_e, gt_e +def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + total_e = set(range(num_edges)) + tp = len(subg_e & gt_e) + tn = len(total_e-(subg_e | gt_e)) + return (tp+tn)/num_edges +def check_retrieval_precision(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(subg_e) +def check_retrieval_recall(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(gt_e) + +# %% +retrieval_stats = {"precision": [], "recall": [], "accuracy": []} +subgs = [] +for subg, gt in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): + retrieval_stats["precision"].append(check_retrieval_precision(subg, gt)) + retrieval_stats["recall"].append(check_retrieval_recall(subg, gt)) + retrieval_stats["accuracy"].append(check_retrieval_accuracy(subg, gt, num_edges)) + subgs.append(subg) + +pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') +torch.save(subgs, 'subg_results.pt') + diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/rag_generate_multihop.py new file mode 100644 index 000000000000..3640dca39166 --- /dev/null +++ b/examples/llm_plus_gnn/rag_generate_multihop.py @@ -0,0 +1,38 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import RAGQueryLoader +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd + + +# %% +triplets = torch.load('wikimultihopqa_full_graph.pt') + +# %% +df = pd.read_csv('wikimultihopqa_cleaned.csv') +questions = df['question_text'][:10] + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=retrieval_via_pcst) + +# %% +subgs = [] +for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): + subgs.append(subg) + +torch.save(subgs, 'subg_results.pt') diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py new file mode 100644 index 000000000000..ee780aff0860 --- /dev/null +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -0,0 +1,92 @@ +from typing import Optional, Union + +import torch +from torch import Tensor + +from torch_geometric.data import FeatureStore +from torch_geometric.distributed import LocalGraphStore +from torch_geometric.sampler import ( + HeteroSamplerOutput, + NeighborSampler, + NodeSamplerInput, + SamplerOutput, +) +from torch_geometric.sampler.neighbor_sampler import NumNeighborsType +from torch_geometric.typing import EdgeTensorType, InputEdges, InputNodes + + +class NeighborSamplingRAGGraphStore(LocalGraphStore): + def __init__(self, feature_store: Optional[FeatureStore] = None, + num_neighbors: NumNeighborsType = [1], **kwargs): + self.feature_store = feature_store + self._num_neighbors = num_neighbors + self.sample_kwargs = kwargs + self._sampler_is_initialized = False + super().__init__() + + def _init_sampler(self): + if self.feature_store is None: + raise AttributeError("Feature store not registered yet.") + self.sampler = NeighborSampler(data=(self.feature_store, self), + num_neighbors=self._num_neighbors, + **self.sample_kwargs) + self._sampler_is_initialized = True + + def register_feature_store(self, feature_store: FeatureStore): + self.feature_store = feature_store + self._sampler_is_initialized = False + + def put_edge_id(self, edge_id: Tensor, *args, **kwargs) -> bool: + ret = super().put_edge_id(edge_id.contiguous(), *args, **kwargs) + self._sampler_is_initialized = False + return ret + + @property + def edge_index(self): + return self.get_edge_index(*self.edge_idx_args, **self.edge_idx_kwargs) + + def put_edge_index(self, edge_index: EdgeTensorType, *args, + **kwargs) -> bool: + ret = super().put_edge_index(edge_index, *args, **kwargs) + # HACK + self.edge_idx_args = args + self.edge_idx_kwargs = kwargs + self._sampler_is_initialized = False + return ret + + @property + def num_neighbors(self): + return self._num_neighbors + + @num_neighbors.setter + def num_neighbors(self, num_neighbors: NumNeighborsType): + self._num_neighbors = num_neighbors + if hasattr(self, 'sampler'): + self.sampler.num_neighbors = num_neighbors + + def sample_subgraph( + self, seed_nodes: InputNodes, seed_edges: InputEdges, + num_neighbors: Optional[NumNeighborsType] = None + ) -> Union[SamplerOutput, HeteroSamplerOutput]: + if not self._sampler_is_initialized: + self._init_sampler() + if num_neighbors is not None: + self.num_neighbors = num_neighbors + + # FIXME: Right now, only input nodes/edges as tensors are be supported + if not isinstance(seed_nodes, Tensor): + raise NotImplementedError + if not isinstance(seed_edges, Tensor): + raise NotImplementedError + device = seed_nodes.device + + # TODO: Call sample_from_edges for seed_edges + seed_edges = self.edge_index.to(device).T[seed_edges.to( + device)].reshape((-1)) + seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) + + seed_nodes = seed_nodes.unique().contiguous() + node_sample_input = NodeSamplerInput(input_id=None, node=seed_nodes) + out = self.sampler.sample_from_nodes(node_sample_input) + + return out diff --git a/torch_geometric/loader/__init__.py b/torch_geometric/loader/__init__.py index 266f498a113b..7e83c35befb6 100644 --- a/torch_geometric/loader/__init__.py +++ b/torch_geometric/loader/__init__.py @@ -22,6 +22,7 @@ from .prefetch import PrefetchLoader from .cache import CachedLoader from .mixin import AffinityMixin +from .rag_loader import RAGQueryLoader __all__ = classes = [ 'DataLoader', @@ -50,6 +51,7 @@ 'PrefetchLoader', 'CachedLoader', 'AffinityMixin', + 'RAGQueryLoader', ] RandomNodeSampler = deprecated( diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py new file mode 100644 index 000000000000..f4e87f83404b --- /dev/null +++ b/torch_geometric/loader/rag_loader.py @@ -0,0 +1,88 @@ +from abc import abstractmethod +from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union + +from torch_geometric.data import Data, FeatureStore, HeteroData +from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput +from torch_geometric.typing import InputEdges, InputNodes + + +class RAGFeatureStore(Protocol): + """Feature store for remote GNN RAG backend.""" + @abstractmethod + def retrieve_seed_nodes(self, query: Any, **kwargs) -> InputNodes: + """Makes a comparison between the query and all the nodes to get all + the closest nodes. Return the indices of the nodes that are to be seeds + for the RAG Sampler. + """ + ... + + @abstractmethod + def retrieve_seed_edges(self, query: Any, **kwargs) -> InputEdges: + """Makes a comparison between the query and all the edges to get all + the closest nodes. Returns the edge indices that are to be the seeds + for the RAG Sampler. + """ + ... + + @abstractmethod + def load_subgraph( + self, sample: Union[SamplerOutput, HeteroSamplerOutput] + ) -> Union[Data, HeteroData]: + """Combines sampled subgraph output with features in a Data object.""" + ... + + +class RAGGraphStore(Protocol): + """Graph store for remote GNN RAG backend.""" + @abstractmethod + def sample_subgraph(self, seed_nodes: InputNodes, seed_edges: InputEdges, + **kwargs) -> Union[SamplerOutput, HeteroSamplerOutput]: + """Sample a subgraph using the seeded nodes and edges.""" + ... + + @abstractmethod + def register_feature_store(self, feature_store: FeatureStore): + """Register a feature store to be used with the sampler. Samplers need + info from the feature store in order to work properly on HeteroGraphs. + """ + ... + + +# TODO: Make compatible with Heterographs + + +class RAGQueryLoader: + def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], + local_filter: Optional[Callable[[Data, Any], Data]] = None, + seed_nodes_kwargs: Optional[Dict[str, Any]] = None, + seed_edges_kwargs: Optional[Dict[str, Any]] = None, + sampler_kwargs: Optional[Dict[str, Any]] = None, + loader_kwargs: Optional[Dict[str, Any]] = None): + fstore, gstore = data + self.feature_store = fstore + self.graph_store = gstore + self.graph_store.register_feature_store(self.feature_store) + self.local_filter = local_filter + self.seed_nodes_kwargs = seed_nodes_kwargs or {} + self.seed_edges_kwargs = seed_edges_kwargs or {} + self.sampler_kwargs = sampler_kwargs or {} + self.loader_kwargs = loader_kwargs or {} + + def query(self, query: Any) -> Data: + """Retrieve a subgraph associated with the query with all its feature + attributes. + """ + seed_nodes = self.feature_store.retrieve_seed_nodes( + query, **self.seed_nodes_kwargs) + seed_edges = self.feature_store.retrieve_seed_edges( + query, **self.seed_edges_kwargs) + + subgraph_sample = self.graph_store.sample_subgraph( + seed_nodes, seed_edges, **self.sampler_kwargs) + + data = self.feature_store.load_subgraph(sample=subgraph_sample, + **self.loader_kwargs) + + if self.local_filter: + data = self.local_filter(data, query) + return data From 79784cb68dec914aa1765322d642984a1086d7b3 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 19:29:40 -0700 Subject: [PATCH 660/752] changelog 0 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704c916c1490..acfe87a1cd70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added `loader.RagQueryLoader` with Remote Backend Example ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `data.LargeGraphIndexer` ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `nn.models.GRetriever` ([#9167](https://github.com/pyg-team/pytorch_geometric/pull/9167)) - Added the `WebQSPDataset` dataset ([#9481](https://github.com/pyg-team/pytorch_geometric/pull/9481)) From 4b0e6587bdb014a14ed4865c9cd41b1f19b92c86 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 20 Aug 2024 22:45:13 -0700 Subject: [PATCH 661/752] RAG Backend documentation 0 --- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 20052 ++++++++++++++++ .../llm_plus_gnn/doc/media/remote_backend.svg | 1 + 2 files changed, 20053 insertions(+) create mode 100644 examples/llm_plus_gnn/doc/1_Retrieval.ipynb create mode 100644 examples/llm_plus_gnn/doc/media/remote_backend.svg diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb new file mode 100644 index 000000000000..72912715646d --- /dev/null +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -0,0 +1,20052 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Retrieval Algorithms and Scaling Retrieval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Motivation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When building a RAG Pipeline for inference, the retrieval component is important for the following reasons:\n", + "1. A given algorithm for retrieving subgraph context can have a marked effect on the hallucination rate of the responses in the model\n", + "2. A given retrieval algorithm needs to be able to scale to larger graphs of millions of nodes and edges in order to be practical for production." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph, and conduct some experiments to evaluate its runtime performance and its effects on hallucination rate." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We want to do so in-line with Pytorch Geometric's in-house framework for remote backends:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABJkAAAEdCAYAAACv07ZCAAABYmlDQ1BJQ0MgUHJvZmlsZQAAKJFtkMFLAlEQxj/LEHSJhIgOBXsLw8LWQLoEZlGBh8WKstu6rmul9tjdCG8dgs5B1H8Q/QGBBB46d4ggSCiKDuE5CPZSss3TarWaxzA/PuZ7b94AXYLCWMELoFiyjNT8jLiWXhd9dQQwjCAEDCqqyeKynKQWfNfOsO/g4fV2jN91XX4IXRzmYvtPr/Xe89rj3/6O8Gc1U6X6QSmpzLAAT4RY3rUY5z3ifoOGIj7irLf4jHOmxdVmz3IqQXxD3KfmlSzxM3E406brbVws7KhfM/DpBa20skR1gHIIs5hDko4IGRJimMAUFmhH/3smm54EtsFQhoEN6MjDInecFIYCNOJFlKBiHGFiCRFKie/69w5dLXsAREfpKb+rbb4A1RMgeOVqI1v0nWngMs0UQ/nZrMf2mrmo1OJABeg5dpy3VcAXAho1x3mvOE7jFOi+J6/9CQKfZRsiTTfTAAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAASZoAMABAAAAAEAAAEdAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdAJw+2YAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjI4NTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xMTc3PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CjP0I9MAAEAASURBVHgB7J0FvBtF14entLi1OC9WoHjRj1K8WHF3p7y4S3Eo7l6KOxR3d3d9sWLFihSXogVa2G+euZ29m9xNskk2uZvkf35Nszs7O/Js7u7smXPOdAmsGIkIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIVEFgvCrO1akiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIi4AhIyaQfggiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIQNUEpGSqGqEKEAEREAEREAEREAEREAEREAEREAEREAERkJJJvwEREAEREAEREAEREAEREAEREAEREAEREIGqCUjJVDVCFSACIiACIiACIiACIiACIiACIiACIiACIiAlk34DIiACIiACIiACIiACIiACIiACIiACIiACVROQkqlqhCpABERABERABERABERABERABERABERABESgW7UIfv/9DxMEQbXF6HwREAEREAEREAERaBoCE040oRm/W9XDrJryGDt2rBkzZmx7HV3aNtu+cnbMuD2boT09Lo0S4tLb08blsAm5aZzYpWOaTbHJHaRLXGKHXEoQAREQAREQARGoN4GqRz8jv/ra/Pvvv/Vut+oTAREQAREQAREQgcwSmHOOnpltm2/Yb7//br7+5ju/2xTfccqnNn1UrqbK7eUpqtrSwNCetz2tPT08GjmYm9aWN0xrP9WVHaZHNiKb5HaSo3Ibl6HtK2cnv7Wu+WF5/qhNaE+j+PaEuPTSaW1l5CFsa3dMYux1cUV0qGlcQ0unuxzt/7XV3dYst+17HJYU2YhshnnDtEgZcdeg7fC43JGT2jZz0wun5VQSe20iRdvMdq/tHydGpEC6vQa557edEncdIoVpUwREQARSIdDFWiFVZYYkBVMq10GFiIAIiIAIiIAINBmB8cbLdlQCLJncILD9P3cFcnfbholt/9vDkY3I5rgrF7QfJmVchpzU8CQOhxnGne9PyU1v24tLy6nElzauLJu/7V9Ytk8YV1LpdDtE7pjXlhIzdI5Nc83LLcHttf/n2tC+W05eV3j7+e27divS7rDIuLScnA5PezHjTgzPT563/RS71aGY9oT2fONqbc9Owjix7c7N6NLjeJMxJqsvSN8FCMQpntr0hLlqKrfX/p8rrX23PW+41X4wLy+7bblys0TS2rO4vG1HwtM4aksIU31x0VITpLWVMq6w6Jcr3RffXsu4LfvVnjaujA5pbekd9a221bknt9Ubl0grYtLbknILcXt5edvSxrXP1TKu3eGpbRu5u3Fp7WXk5m1L75hGPeNSw4ORtPbixuXKzdt+Snt6XFqkGLsZl3dcDnuo/XzSrFhWHdJsSh7CcVk75nQH9F9RAlUrmYqWroMiIAIiIAIiIAIiIAIiIAIi0EkE4hRSbYqrXJWU28vTaLWl0fD2vO1p7enh0cjB3LRCedvSO+aNKPXCg5G09uLGtWxcptwvm6s9ob2Y9rRIMW15cw9x2IpNbPvXtuv/j1XqJVcKJr0GrgWuXeMaN67+9qT29IJprpDwP08lxMNGWEq4kZNaUd62osYVmPOVs9Net9+yh8flaO9th7Rx/emQ3paQe77NG3u9SO6Qc1yd+ipEIKmij/ML53VHwyqcKqv9P5eeu9um7Gr7n4L9qRGFWZjG4ZydcZkjqTmHc3ba6/Zl2MPdunY1M0w/na+05HfV7nIla1AGERABERABERABERABERABEegEAkmtQTqhaapSBDJJIE7x5FRReQqptjS6kKuoiksvntZeRlhSZCOyOY5XRCnYfqptRZgzp0lheofD4xJyvnJ2IiW2p4/bCttCptJpbQ3NQ9hWRqwC0LY6t1CXN/bauHy5mduT2tMLprU1zf4fl7ft4EQ2zmQ5IkumcmgprwiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIQCyBbAcLiG2yEkVABERABERABERABERABERABERABERABLJGQEqmrF0RtUcEREAEREAEREAEREAEREAEREAEREAEGpCAlEwNeNHUZBEQAREQAREQAREQAREQAREQAREQARHIGgEpmbJ2RdQeERABERABERABERABERABERABERABEWhAAlIyNeBFU5NFQAREQAREQAREQAREQAREQAREQAREIGsEpGTK2hVRe0RABERABERABERABERABERABERABESgAQlIydSAF01NFgEREAEREAEREAEREAEREAEREAEREIGsEZCSKWtXRO0RAREQAREQAREQAREQAREQAREQAREQgQYkICVTA140NVkEREAEREAEREAEREAEREAEREAEREAEskagW5oN+t///mcuueSSnCKnmGIK85///MesssoqZoEFFsg5dv3115unnnrK9O3b1wwYMCDnWKmdP/74w/z5559mqqmmKpr11VdfNZdeeqnp0aOHOfHEE13egQMHGs7fc889O7SpaGFFDiZtT5Ei3KGXXnrJXHHFFWaaaaYxxx13XNHst9xyi+P3zjvvmEknndTMPffcZvfddzezzz57h/M+//xzM8sss3RIV4IIiIAIiIAIiEBrErj88svNyy+/XLTzK620ktlkk02K5il2MG4cViy/P/brr7+agw46yO0yHmJcVGv57rvvzJFHHumqOfTQQ82ss86aepVBEJgvv/zSzDTTTKmXrQJFQAREQAREIBME7MMuNbn55psD26nYT7du3YIDDjgg+Pfff8P6dtttN5d3m222CdOSbFjlVDDzzDMHTz75ZMnsN954o6vDKljCvFbh5NLuvffeMK2ajXLaU6qea665xrVtjjnmKJgVhv/9739dvnze448/fvDAAw+E59qBTLD11lsHdpAYpmlDBERABERABERABLbYYovYsUR0bLHvvvtWBSpuHJakwG+++SZs20cffZTklKrzfPDBB2GdVjlWdXn5BViFXrDkkksGxxxzTP4h7YuACIiACIhA0xBI1ZIpqjW78sorjaVkmIm64447zGOPPWZOP/10ZznkrZY22GAD07Nnz7KtiazSxPzzzz/R6gpuL7TQQuaUU04xWFTVSsppTxpteOWVVwyzj127dnXWTiussIL55JNPzNFHH23sAMnsuOOOBssl5LLLLjNWcWVWXHHFNKpWGSIgAiIgAiIgAk1CwE72mdVXX9315uOPPzZW+eG2zzjjjNByaP7556+qt5WOwyabbDI3fqPyqaeeuqo2ZOXkQw45xLzwwgtmjTXWyEqT1A4REAEREAERSJ1AzZRM1jrJjDdeW8invfbay2y55ZYG9zgesChlrGWTmWSSSZwbG65eXnA7u+GGG8zw4cOdkspa9Dgzbe8Whzseyivk7rvvNj///LNZZ511zFVXXWX+/vtvs9FGGzmlCtubbrqpmWCCCTrU4evi+5FHHjGPP/64mW666czGG28cmi+jHKMdCH2ZaKKJ3Pb9999vvvjiC7PYYouZ//u//3PugXHtIfPIkSOdgu2zzz4z8803n1l//fVN9+7dXTn+P5RDt99+u/npp5/Mqquu6pOLfj///PPuOKbWBx98sOO81FJLmcknn9wce+yxznXuxx9/dIonFFIIptmwW2uttZz7Imn08ZlnnjHPPvusY9SvXz+z+OKLc8jJsGHDDHXNNddcrmza2adPH7Peeuu54xyD36hRowzXCR52hm7c2W1fn376qbn11lvdderfv79z6bvzzjud0m+zzTYL8+Jq+fDDD5vffvvNLLrooo6V//2EmbQhAiIgAiIgAiKQGoHlllvO8EFw1/dKpg033NBNAvqKCo2xcCdjHGatlcyHH37oxh/zzjuv4Xnvx21x47CbbrrJnce4iDGVtcB2k4eM5xZeeGFXLWMAQh0gfjzw9NNPm/fee8+FWaB8xhOMb5ZddtkOipvff//djTtpF+MTxoeEI7DW4GarrbZyY1BXeIn/ktbJWPCee+4xuAf+8ssvZsYZZzRMpvbq1cvVwDHGhQhjM8ZkTAp26dKlJEP6wFiVsAfLL7+8ue222wxjNFgzvoaxF9rB+JhxFeNCrsWCCy7o6vF5Ro8ebaz3gXn77bddmSgafTt9Hn2LgAiIgAiIQMUE7MMoNYm6y1lLo5xyrTIjNEG2D0t3LN9dzvrCB9ayKcxnO+W27YMvsMoH52rn0/y3HVi4srwLnH2Yhuc/9NBDQZyZts9rY0GFeSnPzpQFL774oisP02xfBybbXuyD2KUfddRRRdtjlS+Br8eXYwdjwRtvvOGLCh599NHADgDCeuxAI1hiiSXcfjF3OTvQCM+xCpnAzjgGdqAQlus3bFyBMJ9vA+1CMAm3sbJyjlO/VQiGLo2DBw92x2mTtQRz20svvbQ7/7rrrss515d/2mmnueP8Rzv9eRynfKv4c+dxTb2ceuqpgbXKyinPKrwCq7zyWfQtAiIgAiIgAiJQQwKMf/yz3E6A5dTkxzP5Yyw7oRf07t07PM+fzzP+q6++cmXEjcPs5JU7xypZcp7/uPw/+OCD7rw4d7mddtrJnWcVRoGN0ZRTL2MeL9Q9zzzz5By3Cp+A8mmjVWz5rDnfce5ySeokjIFVkOXURz0TTjhh8Oabb7o6rHKow/GxY8cGSRhee+217lw7ERhYhVlOOZTrhbGytVzPOU471l57bZ8lsBOfgbVOy8kz8cQTB4zrJCIgAiIgAiKQBoG6rS4322yz2edcm1gFjt/M+cbFbsSIEc7aBncvrHxwBWOWhxkgvsnjZ7QIynj44YfnlIFlzqBBg9zsEMEqiwmuZRdccIErm5mzH374wQXOLnZO9Fih9jBDtP322zvrJL6x+Nl2220NFk1YdSF2YGEwU8eaiMDnViFmrILGvPvuu9EqYrdhYmMyuWOvvfaaIZA5QdVhzCwkZSPMoDEbiTDbBTvM3qkTNsz+Ybk0dOhQ1y47GDJDhgzpELyd2U2si+BqYzO4gOtYTCHMxFnFmdlhhx3cvlVMuW/cGXfddVc3m4eVFVZKuEved9997rj/zw6+DNeRmUXM85944gnHw8bbMmeeeabPpm8REAEREAEREIFOJpA/xrr66qvduAVLIix4rILIWc9geWPjXpZsLRbuVoHirKiwmh4zZoy58MILS56HhTTjqddffz0c5zB+8XLEEUeY999/31mnM8bBcuqtt95y5fs85X4Xq5O+Yz3EQjeM47799ls3XsLaipAFCGMd73rI+MyPZ8thiAUUi7vw7ReHYQEdxlLI8ccf7yyesHRn0Rss0LHUZwwNZ2T//fc3LBhjJw3dmIuxGeNWxqRY1UtEQAREQAREoGoCaWiqfBnFLJms+1Y4a+JnS/ItmaxSweWxfvgBs1vWXS2wih9ffPjtrV6igb/9LFt+gMq4GTSfl/q9RGfwrDlzkMSSyZ+b3x6rSHH9sC52gTWZdtnov59BY6YMyyN78dwHix8v9uHv0opZMvm8dvAQWNe1gBkoXxbfa665ZmAHai6bHYS4Y8xsebEDIZdmlXVB1ErLKsRcunXtc1m9JRPtxsosX+wgyiVZl7gAyy7qps9ItH9WceTS+G/zzTd3+bwlk105xu0vs8wyYR47GHJpdpAUpjXqBtcWNv4TvdaN2ie1WwREQAREoPkIRMdBhSyZ8sdYULAr/bqxjlVUBFbhEdiVbt0z/OSTT3aQ4sZh3pLJThSGIK2CxJ1nJ6ZcWjFLpuiYIWopT1uQaaed1pVlY1W6ff6zyiuXxlilEkumYnVa1zRXNhbbjMvsBKYbR4aVj9tYeeWVXb78wN+lGHpLJsZj1g3QlYbHgHWTc+V56y8WuaF/UasurOatW6HrM+fSRvLYyb+weVhIkWYnDsM0bYiACIiACIhApQTGsw+Vuoj3Q6cy4vfECdY+WOQQl4cZGKuQcDMw6667bhjIOu68aBp+90mFGERe/OwS+1hTRcXCDXeTBBzHQgphRo6+2sGOi0Xkz8WiyddBbCo7cAnLj7YjTCywQSwDgqoz82QHEeFsHtZCWB8VEmIwIcRXYobLizWndpvEw6LtXoi5kL90MDOEzBQSH8BbUJHfsyIWE0JwciyZvGCFFRXPillAOPHhd4BYc/fQKit6Tta3scayLobO8o6A68wsYvmFlRn7WMDxsYPfrHdF7RMBERABERCBkED+GItnPguRME5jgRXiBTGGQPx4IDw5ZqOnXfzFi1+g5a+//vJJBb/jziMz8TiJ7Wknxty5WFh5yR9/+PSk38XqXGSRRVxsJPpMnCisguacc05nBU5Mp2JSDkPGbMQzRbDqJzg6AjOrqHLxrdj3cbbYxnKd64N1ExZm/roQF9OPu4jvhDA+lYiACIiACIhAtQRqFvg7v2F22VaXxEMRk+g4YfUQTJ/vuusuZ2aN4gRlBSbI008/fQc3rrgy/AM37lh+mleEkB514bOWTjlZowoXTIpLCW5nCG057LDDOmRHMYMpNYJrG4MhTKwRr3RxOwX+I6A55tEE4L7ooosM9TGI4INpNkonOyPpTKHjiph55pldclTxR4IfXBAo0s6WhafmM7Wzi65sgosTcNIugezaQOByri+Cex6CYo0Bp43Z4PYx0Y6KZ4VykUCc+QJ7FHGNIgcccIBz+4u21ysXo2lso3SCJS6bEhEQAREQARHIOoH88QBu9CeccIJhMuqkk05ywbdxx8Jtzo8HivUpGrA6SX5fVrHzUMLQHsY0uK5Z6yF3Wv74w5eV9LtYnUwc4Y625557mltuucWtqEwoAca0BPcmWDl54qQchtE2UFaUGccYU6FsQtnmhVAQCGNsP+ZiH0VY/qp9Pug6xyUiIAIiIAIiUCmBNo1ApWcnOA8lCj781l3I5bYBFMMVR/JPxzd/9913dy/el112mRlhLYrOOussly3q2+8fslHljy/LH/P7xb5ZzYRYQIgv37qeOX93lFp+QOCVUd9//71biYP8fiaIbV+nb49/SLPiCit27Lfffm7lOpQtKG9QKNmAlM7Kh/NZoQ1hUODb4RIK/MeKJSgnWCWF2EW0BUUGAxzvT08MJSS/baR5yyJWdGEgiLAKCzELkKhlFfu+DLYR6kTBxAwYsQawgCK+E+IVKsQM8MpE4jbRN6yfiA8QFZY2Rjh/n332cayw5qJtM8wwg+F6NIJgvQRz4kqVI8Se4HfG+RIREAEREAERyDKB/PEAq5wh1vXdxYe0rnLhpJ0fD3RGf1ZbbTVXrXXVc/E2UbRcfPHFNWvKCy+84MYwNhyEGwcQL5PV3RDGfn4C0fPz40WOp8UQhZOf0GM85/mjTMIafe+99zY9reWYtxjDsonxKemMcxmncVwiAiIgAiIgAtUSqJmJiH+IYbbsFTIoDU488cSCbeY4bnLMtDADhKKHJVaRVVZZJTwPax2scAhCjek2wRMrEQJyo3Dp3r27C7xNGSg6bFwhVxymzpgWb7PNNi4YOUopZojyJa49mIxjbYRLHibJKBEwR2abma5JJ53UzW5hibTLLru4gNjPPfdcaOGUX0d0n8EcAxmUXitY9zOUVlhYeQUTihtvIk7bECybUHgdfPDBYZsYfJGGWTXL2KI4ou+lFCUoobAuwgKL4N7w84M3Bilcc2YSsXDCTeycc85x7WWQl28ltvPOO7uA57jGMTjCGoq+UXYjWfgQfN0H3oxeq6TbuNH5v5Ok5yifCIiACIiACHQmAcYgWAudffbZLtQB4yTGTYgfk3RG+7D4ZrKSSU5c+nm++rFdLdpDaAQbR9SN4Vi4hnAEfvKISTcfJsKPyZh8o20E5E6TIW74TPxdddVVBjc9xtGMdRmTMXZkfM3YmbEZCiaUY4y37r//fmNjZZo99tijFnhUpgiIgAiIQKsRsA/e1CQa+NtydEEEretVYF2hAgJaE/w6KgTeJp9V4oTJVukSWAVJGCSboNoDBgwIrFVQmMeuhBEGLiTAJOKDeVtLoDAfG3EBJ6N5rdWMawP1WGWPC2DpC7BxjQKrwHHH7QPa9cHO+Lh9Ajl7iWuPXcHNBbm2D3SX3w4s3P6oUaP8aS54I323s08uj7WECayZtdu2A5IwX9wGS+JahUzIAY70y7qcBdTt5euvvw4Iss1x6rGrrLhD1sLM9cf3z1rTBHZQFFjTbn9q4AN/k54vdiW8wAeYnHLKKV1ATQJ1U48drLjsLOlLcEsCfFoFYmAHLy6oJHmstVdYpJ3lC2wMo5CDHZAF1s0wPJ71DRubyvWbflX64drAQCICIiACIiACnUEgSeDv/DGWVSgFdrIqsBNP7vnXv3//wD8T7cq5rhtx4zAf+NsqQ8Kunnvuua6MxRZbzKUVC/zNQiVe7ARe+Oz1i61wzFoIBQS0thNhwVprrRXYibwwn1WA+dNzvq3FUZjHxop0x6wFvksrVScLe1gFT2AnWcMy7IRjYN30wjqs4ilgXMxYgfHhxx9/HCRh6AN/Mz6KirVQcmXZMBNhMnlt7CaXbi2nAjtJG9iV8cLjjM1YFAYutIMFW7huBC+XiIAIiIAIiEAaBLpQiH3IZE5w38K6hfhFzLzki1XWGNzRmKGqVnDNwsIG66I4wYwYa6FonKL8fIXag7sgcQEwQcaUOU6sUsjwoY5yBcsq2oflGG50cYJL4AjresiMVlwfaR/WSN76LK6MuDR+OtRN7IP8vtFvlg3m+mCh5AOMY9WEtZgd8LkZvGi5WEBhnUV5jSLMVGKFlJbYQaqb1UyrPJUjAiIgAiIgArUmgKUy4wDGUlkQFkXBNQ33PSx0ECzkCdDNOIgFZmoluKlZ5ZFz+SfGZb4QpBtrJ8aF0ZiTaTPE4p9xXVwbfJsY/xH6oFFCE/h261sEREAERCDbBDKrZMo2NrUuCQH8/Ym1RBDwQw891JnOn3nmmU7phkm3j9OVpKys5rHLLxu7THMY26vadtpZV+OD5Fdbls4XAREQAREQgVYkwEIiuN6jPLGW1+6b0Aq4kK1gXfyY0JGIgAiIgAiIgAjUhoCUTLXhqlItAWINbLvtth3iMmD5w4qBcVZVjQaOVfGqXbEmv88ZNS7Mb6b2RUAEREAERCCTBLDeJvg3QbejQmwkrJwWXHDBaLK2RUAEREAEREAEUiQgJVOKMFVURwK4PRLQHNdHAk+y4pxffa9j7sZL8SsQptlyucylSVNliYAIiIAItCIBQgUQ2BqXMFzYcE8jILdf4a0VmajPIiACIiACIlAPAlIy1YOy6mhKAmnHY/KQpGTyJPQtAiIgAiIgAiIgAiIgAiIgAiLQSATiI1E3Ug/UVhFoIgIssYzySiICIiACIiACIiACIiACIiACIiACjUZASqZGu2Jqb1MTYLVAiQiIgAiIgAiIgAiIgAiIgAiIgAg0IgG5yzXiVVObM0OgFjGZFPg7M5dXDREBERABERABERABERABERABESiDgCyZyoClrCKQT2DRRRfNT9K+CIiACIiACIiACIiACIiACIiACLQkASmZWvKyq9NpEVh33XXTKsqMN9545uCDD06tPBUkAiIgAiIgAiIgAiIgAiIgAiIgAvUkIHe5etJWXU1JYIEFFjDvvPNOKn2Tq1wqGFWICIiACIiACIiACIiACIiACIhAJxCQJVMnQFeVzUXgvPPOS6VDRx11VCrlqBAREAEREAEREAEREAERaCYC//zzj+GTPyH777//Ju7m6NGjE+f1Gcspn/aNGTPGn5r4O2kdSfPlV5zPLP941vbL6Wc+80K/k6z1sdnbIyVTs19h9a/mBFZYYQVTrYJo4MCB5uijj655W1WBCIiACIiACIiACIiACDQagVlnndVMP/305vjjj3dNv+mmm8zCCy9spplmGrPZZpuZP/74o2iXnnrqKUMZSeXXX381W2yxhZlhhhnMHHPMYe69996ip6IYWWeddUw5k8+DBw82eETMNNNMZsCAAabQKtN4TCy55JKmR48epm/fvuaRRx4p2hZ/8P333zfbbLONmWWWWcwSSyxhLrjgAn+owzcMZ5555vCzyiqrdMhT64RqmXMN6MNkk01mhgwZUuvmqvwiBKRkKgJHh0QgKQEURJUqmg466CBz+umnJ61K+URABERABERABERABESg5Qjcd999ZtCgQeaTTz4x++yzj1PovPfee+b33383hx9+eEEe11xzjVl//fXLsjI65JBDzAQTTGA++ugjc8YZZziF07fffhtbx1dffWU22GADc//998cej0v83//+Z04++WTz4IMPmg8++MB8/vnn5qSTTuqQFcXJRhttZFhsaOTIkeacc85xiiPaVUoOPfRQM9tss7k+XHvttebII480w4YNiz3tscceM5dccol56KGH3Oeiiy6KzVfLxGqZE9+Wa7HmmmvWspkqOwEBKZkSQFIWEUhCAEXT448/bhZaaKGS2SeffHKXh/ynnHJKyfzKIAIiIAIiIAIiIAIiIAIiYAxWTP379zfLLrusmW666cwxxxxjhg4dGovmhhtucMqbM888M/Z4ocQrrrjCTSAzZkeB1K9fP1dvXP7VVlvNzD333Ga99daLOxyb1rVrV3P11VeHljcokYYPH94hL8ok0lGuYaGDJdPSSy9d0rLqr7/+cpZPeEtMOOGEZq655jIrrbSSueWWWzrUgfIKKyr6Mf/887vPnHPO2SFfrRNqzbzW7Vf57QSkZGpnoS0RqJoArnNvvPGGeygxwxInffr0Mfvvv7/zKSe/RAREQAREQAREQAREQAREIBmBESNGOKWJz92rVy/zww8/mN9++80nhd+rrrqqeeutt5y7WJhYYuObb74xf//9t5l99tnDnNRBvXGCFdBpp53mLJ/ijsel4eqHogwFz9lnn21Qhu22224dsuIOONFEE5kuXbqEx3ANjFNIhRnsBoqlyy67zCmaSMcVDaup//u//4tmc9uvvfaamXLKKZ21Fko7lE0ffvhhh3y1TKgH81q2X2XnEpCSKZeH9kQgFQJYNfHAuP7661153OgxiSXw3ksvvWQ4LhEBERABERABERABERABESiPAIqZ7t27hydNMcUUbhtFU75MNdVUBjeqcoTyUbpEFTvUEVc+5aIIqlSeffZZgxsgiiSCVucLcZhQ+jBBjWsg1k+c8/333+dnLbiPUmrDDTc0KNyIG5UvP//8s7PEOuKII1zZsF133XXN2LFj87PWbL+ezGvWCRUcEijvLy48TRsiIAKlCOD/TcDAXXfd1RA879VXXy11io6LgAiIgAiIgAiIgAiIgAgUIYDiJWq1xDYKIQKDpyH55VMmdRAEPG3ZdNNNXQwkApoTfDtuZTVit44aNcopiR599FGz9957u1hLSdqCMmrFFVc00047rSEuU5xstdVWBmusBRdc0FmInXrqqU6hReDwekk9mderT61cj5RMrXz11feaEbjqqqtcUL499tjDreSAaSoB/iQiIAIiIAIiIAIiIAIiIAKVE2C1tE8//TQsgG0UTFgDpSEzzjijsyr6+uuvw+Koo2fPnuF+tRsPP/ywU+z4coj59N1335kvvvjCJ+V8s7rdZ599ZnjHIPB5kphJWActv/zybmU6Jr/HH3/8nDL9zhNPPGGefvppv+tc7Qh6npbSLiy4yEY9mBepXodSJiAlU8pAVZwIXH755W4ZUmIynXvuuQ4ISiYC9/HwkIiACIiACIiACIiACIiACFRGYMsttzS33367i7U0evRog+XNJpts4goj4DVWObiIlZJ33nnHvPvuux2yoaxiRbcTTzzRxWZ68cUXnRJmjTXWcHlxW3v77bc7nJefQDBt2sJ3vowZM8bsvPPObmU83OSGDBniFEco0PL7sPXWW4cBu4kvhbucDzJeqA9YROEihxUTgdF/+eUXZw0FLyTaB45tt912Lm4TLnKE/EDphRtgsT5QziuvvJITq4pYUbTRC6u9Pffcc37XFGpvWszDirTRqQS6dWrtqjyzBNBoI3wTnJqPpDQBlv7kgYHfNMudellsscXcJi5zq6++uk/WtwiIgAiIgAiIgAiIgAiIQBkEWAFtv/32c8G8p556atO7d29z/vnnuxJwD1t55ZWdEmWeeeYpWqpf4RnroHwhliqKHKyXWAmOcf2ss87qsrGN+5yPvZp/rt9HwUJbPv7445wg4hxfc801w1WpUeTgLnbbbbc5t7/8PhBUHO+IY4891nTr1s3wvuGtjAr1Abc64sDy8Wyol+Di7Ef7QJwmVrwmGDkKLoJ/s4IfUqwPHN9rr71cnKfDDjuMXXPOOee4c2699Va3T7yp4447LlREFWovmdNg7irVf51OoIsNRBx0eivUgEwQIBj13XffXdCta5FFFnE3WwWtjr9cF1xwgdl9993NgQce6GZU8nMxM0F8psMPPzz/kPZFQAREQAREQAREQAREoOEI1GtieqaZZnLWS0sssUTICIUIljnRIODhwQQbuJ9ddNFF5oQTTiiY+9tvv3XxjKJBwAtmjjnA5PPgwYPNxBNPHHPUuODaBBT3SqPYTOMSf/zxR0Mg86gk6UM0f7FtLKp++umnDoHMS/WhWJn5x5K0t1rmWKFhiUXsKknnEJC7XOdwz1StPBxQIGFKWSxu0Ouvv+7yLLTQQs7CKVOd6OTG4BaHgunQQw+NVTDRPFzmFPy7ky+UqhcBERABERABERABEaiKABPOuGGheOHbu2T5tEX7LG1qMSmNdczzzz8ftp3VmytVMFEIcYiw7CkmWPVUqmDC2ol3rEIKJurFMimJgom8+Qom0pL0gXxJBIut/JXykvQhSdk+T5L2Vsoc2xmstLAck3QuAVkydS7/Tq+dBwDKpUrkqKOOqskDpJK2dOY5zE7su+++hmU/MQctJBy77LLLQnPRQvmULgIiIAIiIAIiIAIiIAJZI8DENO8NfJueaxnTY+62JnYf9z1quDE/2Q8y4l6T5rsC7l2//vqrWXrppd0qa22V6H8RaCeAksm/1xI/q2/fvu0HtVVXAlIy1RV3tiqrRsHke4L/bivHazrzzDPNwIEDEz1EWRVi7bXXNl9++aVhBQWJCIiACIiACIiACIiACDQCgfC9AYXS7FbB5BVLhRpvFU7dv3vYjPpiWKJxcqFilC4CItB4BKRkarxrllqLKzX9zG9Aq4b1YiWLgw8+2AXhGzRoUD6WDvssg4pyibhXKJskIiACIiACIiACIiACIpB1AqGCCeul2cscw35yj7NqavWJ6axfY7VPBNIkoJhMadJsoLIOOeSQVFqLogqz1VaTk046ySmYWNo0iYIJPjPMMINboUJxmVrt16L+ioAIiIAIiIAIiEDjEnAuSJUomOgySilr9eTdmBqXglouAiKQlICUTElJNVE+/Kj98pHVdgsrJgLwOd/sagtrkPOPP/54wzKdMCTQdzmi4N/l0FJeERABERABERABERCBziSAFVP3mXuXb8EUbbR1r+NdgbIkIiACzU9ASqbmv8YdethKCqEOna8ygYcjlkunn366Oeigg8ouDSVTsRX8yi5QJ4iACIiACIiACIiACIhADQgw7sUCadS0/asrnfhNi+7XHjS8utJ0tgiIQMYJSMmU8QtUi+b98ccfqReblvtd6g1LsUCUSzxozzrrLBfsu5KiF1tsMTNy5Ejz+eefV3K6zhEBERABERABERABERCB+hIoFeQ7SWvSKCNJPcojAiLQ6QSkZOr0S1D/Bjz22GOpVzp69OjUy8xSgbjH4SY3ZMgQs++++1bcNCyZEMVlqhihThQBERABERABERABEagXAWIxpSS43cmjIiWYKkYEMkxASqYMX5xaNa0WCo4333yzVs3t9HJZQY5A3+eff77Zc889q2rPNNNMY3r16iUlU1UUdbIIiIAIiIAIiIAIiECtCaQdrHvUb3+bJ598stbNVvkiIAKdTEBKpk6+AJ1RvbemSbPuhRdeOM3iMlPWwIEDzamnnmouuugis9tuu6XSLsVlSgWjChEBERABERABERABEag1gR42nlJa0n0uM+q3v9IqTeWIgAhklICUTHkXphVMOFdZZZW8Xle/u+KKK1ZfSMZKwC3uzDPPNJdeeqnZeeedU2sdSqZaWJOl1kAVJAIiIAIiIAIiIAIi0PIEFll8KWN+Gp4qh/XWWjXV8lSYCIhA9gi0vJIJpdJSSy1l5p9/ftOlSxeDsoRvPosvvnhTLrW5+uqrp/5LnHLKKVMvszML3GuvvczgwYPNlVdeaXbYYYdUm0Lw72+++cZ88sknqZarwqonwCoq/u9f3233QXEQB/0G2n8D1d9lVIIIiIAINA6B7pNNaMyoD9Jr8Ih70ytLJYmACGSWQLfMtqzGDUO5tP/++5vXXnutYE1Ym/DBH/moo45qSoVTwc6XeWCFFVYo84zsZt99993NBRdcYIYOHWq23nrr1Bvq3RX5bc0+++ypl68CKyfA3zq/5X79+lVeiM4UARFoSgLcHxg7NNPzrikvlDolAiKQGgHGQ69/eKsZlVqJRvfQFFmqKBHIKoGWVDJhrVBuIDvy//TTT866JasXM2m7GCAfccQRbrW0pOcUy3fggQc2zQMDt7hLLrnEXHfddWaLLbYo1u2Kj3Xv3t3MO++8ToG58cYbV1yOTqwNAQZU3CMkIiACIuAJoFwqd9zgz9W3CIiACDQqAd4Z3L1vWusy133u6rrxyT3ufMqUiIAINDeB8Zq7ex17V4mCyZdyzjnnmCWWWMLvNvT3cccd51wE0+gEgbGbQXCLQ8F044031kzB5Dkp+LcnoW8REAEREAEREAEREIEsEkAh5JRCn6Tg5mZd5fAMkYiACDQ/gZZTMlU7E/nyyy+bQYMGNcUv47zzzqu6H83ysBgwYIC5/PLLza233mo23XTTqrmUKkDBv0sR0nEREAEREAEREAEREIHOJvD444/buEzWkmmcJVJF7bHnoqySpXhF9HSSCDQcgZZSMu20006pXKDjjz/exWVIpbBOLISbfTVKomaJU7XNNtuYq666ytxxxx1mww03rMsVIfj3Dz/8YD74IMVginVpuSoRAREQAREQAREQARFoJQJO0UTQ7koUTZwjK6ZW+rmoryJgWkbJhOacpejTkkMOOSStojq1HLgEQWB22WWXxO1YZJFFDA+bZpiNIO7Stddea+6++26z3nrrJWZQbcZo8O9qy9L5IiACIiACIiACIiACIlArAuHEdDmKJmv91P0D6zVhz+G9gTIkIiACrUGgZZRMf/zxR6pX9MUXX0y1vM4u7MILL3RWTX379i3YFBQjWC+xIl8zPChwi7vlllvMvffea9Zee+2C/a7Fgckmm8z07t3bBf+uRfkqUwREQAREQAREQAREQATSIsDkslMW9fzdmMd3a7NqwkoJVzovbPN57Sz3WaTXNG4yuxneG3wX9S0CIlCaQLfSWZojx3333Zd6R5ptKWMeHlHrpGWXXdZ8+eWXLlZRsz0cNtpoI6dc4nfRv3//1H8bSQpU8O8klJRHBERABERABERABEQgCwR4H+DD+8KTTz7ZFj4E66Y8Ic9RR13k8uYd0q4IiEALEGgZS6a333479cuJkqmZZcsttzQ//vhj0z0g1l9/ffPggw+azlQw8bshLtOrr77azD8h9U0EREAEREAEREAERKDJCKBkwqrpn3/+MUsuuaTp2rWrmXvuuV0aYTjkHtdkF1zdEYEyCbSEkqlWyiA0+M0ss846q/n555/NTz/91BTd5KGHWxwPPlzkVlpppU7tF5ZM8H333Xc7tR2qXAREQAREQAREQAREQATKITBmzBiz5pprmuHDh5u99trLfPvtt003MV0OD+UVARFoJ9ASSiZMNmsh/fr1q0WxmSlzlllmcW35/PPPM9OmShsyduxY9yB87rnnnAVTFq6dgn9XejV1ngiIgAiIgAiIgAiIQGcRGD16tBtX/+9//3Pj6o033tiMGjXKvPfee53VJNUrAiKQIQItoWTKEO+GagqWTMhnn33WUO3Ob+yff/7pHoSvvPKKexAus8wy+Vk6ZX+iiSYyrNQnl7lOwa9KRUAEREAEREAEREAEyiTw22+/mbXWWssQioTQEywapInTMiEquwg0OYGWUTItvvjiqV/KaJDs1AvPQIE9evQwU0wxhWlkS6bff//dKZjeeust9yDEbzxLouDfWboaaosIiIAIiIAIiIAIiEAhAoR5wEXugw8+cONq/37FxOnCCy+sidNC4JQuAi1GoGWUTGjcJ5100ha7vNV3F2umRrVk+uWXX9yDEF9xYjD16dOneiApl6Dg3ykDVXEiIAIiIAIiIAIiIAKpE2AxIBRMvBdgwYQ1flSYOJV1fpSItkWgdQm0jJKJuExYtaQlRx11VFpFZboc4jI1oiUTwcp5EI4YMcI9CFHmZFF4IPO7xNJKIgIiIAIiIAIiIAIiIAJZI0BQb8bVX3/9tRtXL7jggh2aKOv8DkiUIAItS6CllExpKYYWWmgh0+yucv4vohEtmb777jv3IPzyyy/dg5DrlVXhgdylSxfN/GT1AqldIiACIiACIiACItDCBL766is3rmYC9/777zfzzz9/LA3GtMRrGjZsWOxxJYqACLQOgZZRMnFJUQwRnA6/4Wrk0EMPreb0hjq30SyZmGFhpuWHH35wCqYFFlgg07y7devmgiWyOodEBERABERABERABERABLJC4IsvvnDj6j/++MONq+eee+6CTdPEaUE0OiACLUegpZRMXN0XXnjBxJl4Jr3yxHXaYostzOabb24efvjhpKc1bL5GsmQaOXKkexAyi0IMpnnnnbchuPNQlg97Q1yqxI1kGV9iFZx33nmJz4lm/Pvvv03Pnj3Dz0UXXRQefuONN8yBBx5o5ptvPrPSSiuZwYMHh8eSblTbPl9PNW25/vrrzXLLLWe4x6y99trmiiuu8MVW9F1NW26//XbTv39/M/vss5stt9zS3HDDDRW1Ie6k119/3cwxxxyG/iaRl19+2Wy44YaGgfxcc83ltl977bUkp2YiDxakp556qmGRBe5tAwcOTOyq/tJLL5l11lnH/e75bWB9/Ndff4X92muvvcK/iVVXXTVM14YIiIAIiED6BAg5wcTt2LFjnYJpzjnnLFoJE6eKNVoUkQ6KQMsQaDklE1eWgezhhx9e1kVmxYTHH3/cmYFedtllLk4Rg1wG0eeff775888/yyqvUTLzAvjvv/9mPvg3QQh5EPJyTjBCXs4aRfRAbpQrlaydvBSjhEbpUel9IQgC8+mnnzrlxP777x8GrSdt2WWXNU899ZTZaaedzGyzzWb23Xdfs88++yRrnM2VRvuorJq2oNRBmfPPP/+YPfbYw33vuOOOplIr0WracsoppzhFzq+//ura8v3337uJBBQl1QrXf6uttjKffPKJe3aUKu+xxx5zyplnnnnGLQ/NghVss2jBE088Uer0Tj/OiwgKQxSf6667rllttdXMxRdfbFZZZRUzevToou1DOUXsxHfeeccMGDDA9OrVy5xxxhlm9dVXD/+O4MHfw/jjj2/ILxEBERABEagNgY8++sg9h7p27erG1Ux8JRFNnCahpDwi0AIE7MtMy4pVGgVLLLFEYC9zzme88cbL2bezqbGM7IteYAfDLu9kk00W2Be94M0334zN26iJ9iHj+mdfdDLbBdpo3eICG3spsC+bmW1noYa98sorjrF1mSuURel1IsC9oNDfe5ImPPfcc4G1MApsnC13TU8//fQkp3XIY5UT7nz7Qp1zzCq2A2sVE9gX9jB90KBBrr73338/TCu0kVb7KL/StlhFhGNkXZdzmmmtdwI7CxrYmA856Ul2Km2LnaUNJpxwwsBatwZjxowJqzrrrLMCngOPPPJImFbJxt577x1MMskk7lpaZUvJInr37h1Ya9nArowZ5rXBVl0brTI6TMvqxoknnuh+i++9917YxEcffdT1/5JLLgnT4jZ23nlnx8oq+cLD55xzjjv3nnvuCdPY6Nevn7vn5yTWeIfxAvcHviUiIAIi0MwEGE9Ya9rAKowCq9Avq6vW8jqwYUnKOkeZRUAEmo9AS1oyed0hs6YvvviisZfVWSnZl0tnnm9f2ty+HUy6Y8RyihPM+XHxIA4QM/B33323Icg0s6233XZb3CkNl4YlE4KlUBblgw8+cBZM9kXRzbT49maxrYXaxKwPM/NymStEqDHSn332WWdlhKVQMdcoq1Q0t9xyi7GKhLI6Nnz4cPPQQw+ZbbbZJieu3K677uqCx2NhWUyStq9YGf5YNW355ptvnHXKcccd54tz394k/+23385JL7VTTVtgwvXaeuutDWb+XrCyIiB/NS58Dz74oHOXPPnkk32xRb9x88XyCXfsySefPMw77bTTOosgVqDEqjTLMmTIEGMVQGaeeeYJm4lLJ67LVskUpuVv8AyeaaaZDKymnnrq8DC/CQSXQ4kIiIAIiEDtCbz77rtuXN29e3c3rp5xxhnLqpQxLc8y3bfLwqbMItB0BNpH1U3XtfI6hMKJTyUy/fTTm8MOO8x9br75ZnP55ZebjTbayJn777DDDoYPLwqNKLx4Mfj//PPPM9d8O1vu4nf06NHD3HXXXWaGGWbIXBuTNoiHsoJ/J6WVzXyYlFvLJbPLLrsYlvotJMRpuvLKKw1KA2u5Uihbh3SUUwiLF0TlP//5j5l55pmNnXmMJnfYTtq+DifGJFTTFtp75pln5pSK8uSOO+4wxLxbfPHFc46V2qmmLQQ0RfLv/dNNN52hnb7sUm3IP87CA9tvv72LR7T88svnH47dt9awBqV5vlgLKxdLEAW6ta7KP5yZfdzXWIEIJWi+WIthc+edd+Ynh/so9I488shw32/4yRoUVxIREAEREIHaEmBcgqsziiUmzqNK/6Q1RydOiU0pEQERaE0C2R2xNuj12GSTTdzynmjw11hjDcNsPS8sKJqsq0pD9oqXm6xZMmHtwCz3NNNM42ZaGlnBxI+Ch7IsmRryzyNsNIGO99tvP2Pdo8K0uI2NN97YnHTSSQbldDniFVf85vNlqqmmchaV+enR/aTti55TaLvatvhyUfJgiUWsBxZlIGA/VonlSDVtISA3Qr1RYaCNYp3lmisR4mUxsZBvrVVJWddcc41TWm677baVnF63c0pdh59//rlkXCbfWCyDeTlh8ubss892FoL+mL5FQAREQATSJ8B7C54YrCpNbNNKFEy+VYo16knoWwRal4CUTDW69gQKt/EkzHfffee+sVJZZpllDLPaWDE0kvDAyZIlEwGVUTBhYcWDMO6lu5H40lb/QM66O0yjcc1iexnEHXLIIWVbN+JOhWC5ly+YtfuX/PxjtdhPqy0EeWZgS7BthADZuE6VI9W0BVcuBtK4GtrYQa5aFOrHHHOM2y4VrDqunViycl9COTTBBBPEZUmcRpt22203p4SuNCh64sqqzFjqOlA8z8NSgjIKV0Pc0LFwwjqKpbMlIiACIiACtSHAJCfjalaP4/nFmKIakXV+NfR0rgg0BwEpmWp8HbFqYNlllqC+//77nUsXbhRY3rDC3ccff1zjFlRffJYsmVDW8SBkqXGsD+JeuKvvcf1L4IHMSluyZqo/+0apEWslJO6F+/fff69aoVEOh7TawgqdWDChSCAOEvfGCy+8sJymmGrawrmXXnqps1hiBTRW68O6CQum9dZbz0wxxRRltYXVeFjp7/jjjzc2mHhZ5+ZnvuGGG9yssg2+6p4dxG3LspS6DrQ9idJtyimndK7DWLmhsGP1P34XEhEQAREQgfQJsOI24+r555/fKZhw3a5WvHV+uZNG1dar80VABLJDQEqmOl4LlmK+6aabDC8iuM8xe86swaabbmoeeOCBOrakvKqyYsn08ssvuwchQWSZaSn3BbC8Xtc3N5ZvE088sZRM9cXeULURIwj58ccfO7SbtGpnHjsUWiQh7bagWMB6iL8BYjOVI9W2Zf311zdYR9qV0Yxd4c4QV4/7C0zLjaVnVwM0dvU8Z1WGtRqfwYMHu+4QX4j9uOuX319cxFC6Lb300ubpp58uux355dVjv9R1oA1c56RCPEACsuPmyQQNAdolIiACIiAC6REgjAcKpkUXXdQ993gGpyEomXgWauI0DZoqQwQak4CUTJ1w3ZgpP+GEE5w7gF+djvhNxKBgdZ44S4VOaGZYJZZMBLL99ddfw7R6b2DtwIOQ1ft4ASwV96be7UujPpkXp0GxecsguDfy4Ycf5nSS+wVBl/nbqJdU0xasOlG25PcDJRnBRuOCXxfrVzVtoVzcvFCk44521llnmQ022MC57L355puGgNXlCINqVobDJRorHD5eafbMM8+4/VL3UZRtxPYigDaTD+UoZsppa9p5uX7MgOdfV+ohba655nJKxLh6cZek3yiT8mW++eZzzx5WJZSIgAiIgAikQ4AJDMbVLCbCuDqJpWnSmjVxmpSU8olA8xKQkqmTr+2AAQPMU089ZVhKGyXD3nvv7Wat+WZ2PQvCCxjSWXGZeDnjQdinTx/3ICw3MHAWGCZpg4/LlCSv8rQeAe4PuE5df/31OZ1n1S5iB2GFUy+ppi2//PKLc4HCcigqTz75pHMfLnd1uWragvLcuy5H23Ldddc5xUa5wbZx4SU2VvTz+OOPu6JZUY90XPIKyQUXXGCOPvpop2S66qqrUh30F6ozzfTNN9/crfQZnSjB7Y2XmWK/T5RTJ598snv+4TbsBQUglmUsnsFkh0QEREAERKB6Ak888YQbVxMnludWLVYu1Zi2+uukEkSgkQlIyZSRq4dbBO5zvIQMGjTIzWBj2YSLHYPszhQ/uO+MFeZ48UTBRNB0HoS4UDSr8LJMzKm///67WbuoflkCKBG41nEWH8UAEQT5gAMOMI899pjBLev99983uGDtueeeZrPNNnP3Cn/+scce6+oo1yqI84kVR/uK/b1X05Zll13WrLDCCi4wNu3EYggFDAr3SSed1Fl5+n7Uui0E/Saw9kUXXeTuvzBF0bPHHns4yybuO16StMXnTfLN/QzOXmnIvZ/V1GCAIp368j/eZYzrzrm33357yap4hvDJl0LpxMnKz5+0voEDBxrig7EENvcyXDHYZvXAI488MmxCft8nmmgic+CBB7q/if/+978uThcxqWgLiiYszCQiIAIiIALVE3jkkUfcuJo4hHfddVf1BRYogWcUzwGJCIhAaxJo3jf2Br2exADxsTwY2KN4ImYTLnbEceJT7tLn1aKgTbz4FHvprLaOuPN5mV5nnXVM//79Q5eTuHzNksYDGcGHfamllmqWbqkfeQQ+/fRTN/D6888/846U3t1pp51cUGoUVbx446LE38gZZ5yRc7KvI2pRkpOhyA4rvDEwLNW+StvStWtXpzgnQPZRRx3lPjQHC6ZbbrnFuVX55tW6LdRz0EEHuRXudtxxR1ct91oUdzCOStK2RM8ptk1sJjijXEJwFRs1apTbxqonTg4++GCngMK9jHOTrNaGEi9OiqXnz2onrY94efRju+22c0owgpWvvPLKLph71MU5v++0D978Ngj0ffXVV7sm4z6JEg4LKYkIiIAIiEB1BFi5E8U/n1pPYDOmZZXtMWPGmKwvXFEdVZ0tAiIQS8BG/pdknIB9GQjsC1lgY32wtndgB/CBdbGra6vty0NwxBFH1K1O+yAM7Gx+YF0s6lZnFiriGtu4XFloSku2gb8vq/jo9L5bBY/7W7cWS7FtsYO24L333gus1Vvs8WoTbfycwLq1JSqmmrZQx+uvvx5YJUbBuurVFhvzJxg+fHjBdnCgnLYULSiFg3bCIbAWQSmUlKyIcuuzK6cGdpW+ZIVHcllrreDtt98O7KRGJDV3s1+/fsECCyyQm1jjPev26P4m+ZaIgAiIQKMR4HlhJxACa/lcl6YPGzbM3TOff/75utSnSkRABLJFQO5ysaq3bCWyFDarDTFrfd5555m33nrL4EeNKweBZeshuMzVy5KJmXBc5FhC/NZbb61H9zJTh8yLM3MpMt0Q3EbnmWeemswO3njjjYZPGRLgAABAAElEQVRg2gSwTiLVtIU6CBCK21qc1LMtxP0hOHUhKbcthcpJI90O3g2x6upl8VhJfbPPPntFKx4SfJaltH0swDR4qQwREAERaGUCd999t1lrrbWcVSiuyPUQOxHgxhFaYa4etFWHCGSPgJRM2bsmBVtEnI7dd9/duVM99NBDbhCO+xwvR6yMVEn8lYKV5R1gwF+PwN/33HOPUzDhIshLXauJAiW22hUv3l/cx3AXvemmm4pnTPEof+sMSLMgakv8VcB9mXtljx494jOknFrv+go1n1VZ+XvIyqIYhdqpdBEQARHICgFi9+Eex4ql1157bV2bpTFtXXGrMhHIFAEpmTJ1OZI3hoE2sxEjRowwu+66q4thwcpTG220kVuBLXlJyXLWw5KJVbKIL7P11lsbVndqRcGSiVgplcTSaUVezdpnYtMQF4iBIXFuCln61KL/LEKAUiELorbEXwWshHr16hV/sAap9a6vUBdmmmkm9/fAM2LLLbcslE3pIiACIiAClgATVazsuf3224ex7uoJRtb59aStukQgWwS6Zas5ak25BFgOmxWa+AwdOtQFCsckFhc7rJxYqSep20uxumttyYRb3MYbb+wCxl555ZXFmtLUx3ggI5gXL7fcck3dV3WuMAFc0GxsrsIZdEQEWpAAKxDykYiACIiACBQngDcAiyawqMUll1xSPHONjjKmZfXY0aNHm4knnrhGtahYERCBLBKQJVMWr0qFbcIU9oknnjA2yJ7p27evW+YcVzqW4652GVEsmWyQYfPll19W2LrCp+EKhIIJpVgrK5ggRJwdXGCqvV6FaeuICIiACIiACIiACIhAsxLAGwAF0y677NJpCibYRidOm5W1+iUCIhBPQEqmeC4Nnbrkkku6hwqBwrFwevTRR92NftVVV604zhGWTEjacZlYntqudGF23nlnc+mllzY097Qaz0NZgRLToqlyREAEREAEREAERKA1CFx99dVmq622chPMF154Yad22k+cakzbqZdBlYtApxCQkqlTsNen0qmmmsoceOCBxi51bgj8R+BwZjaIr3Hcccclsko6+uijXRkE4kZ4cLGiERZTfKqRa665xsXV2G233cxFF11UTVFNda4CJTbV5VRnREAEREAEREAERKDmBK644goXdmLvvfc25557bs3rS1KBxrRJKCmPCDQfASmZmu+axvZo/fXXdytGvf322y4I4BlnnGEIorrtttuaJ598ssM5KJe6dOlijjnmGHP66aeHq/l89NFH5oUXXjArrrii+2A1VYmyCbc43PsIbnz++ed3qL+VE7Bkeuedd8wvv/zSyhjUdxEQAREQAREQAREQgQQE8AYgDut+++1nBg8enOCM+mRhTKsQEPVhrVpEIEsEpGTK0tWoQ1vmn39+g4IJV7oLLrjAvPvuu2aFFVZw1kkEBnz88cfNQgst5JRLSZrz4osvOmUTSqmkctlll7mVLvbdd18FN46BJh/2GCgNkkSgzZVWWskQw2zhhRc2WOl98803Zbee+Gc9e/YMP1FLP5Zvx0Jxvvnmc3WVM5hMq335HRo1apRZZJFFzHnnnZd/qMP+Aw88YNZcc82CH2812eHEhAnltCWuyIsvvthxZ+XOSuXll192ynxW/Jxrrrnc9muvvVZ2ca+//rqZY445DG7FjSLE7Tv11FMNExDcywYOHGh+//33RM3nd3/UUUeZZZdd1v0Nrbbaaubpp58Oz91rr73CvwncvyWtTeCff/4xfIIgCEGwHd0PD8RsJM0Xc6r5888/45JTS+Nv4d9//01UXtJ80cLK7XsldUTrq3a7nPrz88b9TqptT9bOxy1up512MgcccIALtJ2l9vEcYIL7119/zVKz1BYREIEaE5CSqcaAs1r8+OOPb3bddVfDyxAxm3iRIS4SL8hvvfVW2c3G4imJookXOFa64MXjrLPOKrueVjhhzjnnNNNOO61mfhrsYvP7xx31q6++cr9xlEBY7KF8QalbjvAC8Omnn7q/y/3339/06dPHnU4aL+BPPfWUG1CyuiTK2n322adk8Wm2L1rZX3/95fqN8ivJi9cff/zhGMEp+kGpc//995s333wzWnxZ2+W2Jb/w999/380Cw3nMmDH5hxPtP/bYY07B8swzzxhW+uTDNtewHKtPWOKe/Mknn5jffvstUd2dnWns2LFm7bXXdrPo6667rkFJxD1/lVVWcasLFWsfL9VMeDAJwuqo2223nfst9O/fP3TthiV/Dzy/arEIRbH26Vi2CHzwwQfudzDzzDO7MQytO/LII909c4YZZnAhAQq1GEX36quvbmaccUb3e33kkUcKZY1NP/nkk909L/agTVx00UWdpTjW4v6zwAILFMreIf3bb781vXr1SnQvxLp8yimnTDxuq6Tvu+++u9lkk006tLMeCSgmtthiC8M1ZZx67733Fqz24YcfNvPOO6/Lu95665nhw4c7RR2/kckmm6xpJzWZ3GFC65BDDjGnnXZaQT6ddUATp51FXvWKQCcTsC8zEhFwBOyMO9OBVX3sLHRBmtYtzpV90EEHFcyjA20E7AA4sAMr4agzAX7/xX7DhZpjrZWC8cYbL1h++eVzstx5553uN29fjHPSS+1YBUPsedZ6I7AD7cAuBxwWMWjQoMC6tgZWQRKm5W+k3T5f/nPPPRdYZZqrH3bWtdYfKvvbmvgHk0wySWAtfso+lxOqbYtVcgR2MOzaQF/sC0pF7ejdu3cw6aSTBtbdNTzfvjQGNiZeYGNThGmlNmxMjbAtVlFTKnsmjp944onut2DjAIbtsZMY7rdsLWXDtLgNjsPdrooUHv7xxx8dS1hEpV+/foF9aY8m1XzbWvm69vEt6XwC/H3aJdHDhthVagOr3AmsUjawFoCBVdIEd9xxR3jcb3BvtQqLwCorAmvxEth4lYFVNgX8/ZcSzj300EPdvd4qMQpm515sLTfcZ9iwYa5d3N+SyEsvvRRYi3P3Wyt1L7RK3WDppZcOunbtGljlfMniK+n7PffcE1ilbrDhhhuWLL8WGayCK7BhHdz99Lbbbgsmn3zygOdZvpDGMf4+7QRBcPjhhwcbbbRRmI32W6vfcL9ZNugT9036m2WxE6dVjQ+y3De1TQREIJ6ALJns3VlinIktM4PVChZNcTJkyBDDbNhhhx1mTjnllLgsSosQUKDECIwG2CSG1n/+8x/nBhptLlYd3bt3D2OaceyVV14xt9xyS9kxt5iVfeihh1wss4kmmiisBotE4qfhhlpIymlfoTLy05999llnVYX1ULXuXLhE4fZ30kknOcuv/LpK7afRFqtcNCNHjnSWMqXqK3QciyP7Iudm3u0LT5gNy0SserASzXflCDNFNh588EHneojFRCMJ93mrADKsKOQF61isC3DHLibMwPP3gtWClx49ehirKHCulT5N3yIQR+Dyyy93MR57WjdjXJW5Lw4dOrRDVix/tt56a/eb4r5JvErcqbiHlBKsRvkbti/0RbPiJktoAj7EsMQqkntbKeH+w9+AnYhz1syl8h9//PFmiSWWcJZMpfJyvNy+Y1GF5SBBpDtLCGTNvZn76QYbbODuL1ah2KE53F+sUslZQ3br1s2NNWl7MwveAFgxY8HHbyHLwphWcZmyfIXUNhFIn4CUTOkzbbgSceHARSEtyXebO/vss90gxVpcmBNOOCGtapq6HMyLUSrYmfym7mezdA43n88//9wMGDAgp0t2FtsQI4gXHy+YtuN68Nlnn/mkRN8op5C+ffvm5Ee5hTsArl6FpJz2FSojP93OnrtFAXjpym9Tft5i+8Tr2X777Z2LyR577FEsa8Fj1bYFJRdxhFDUTTPNNAXrKXUAlwyU9fkKFV4yedkkVpe1eCtazA8//OB44FJsLeOK5s3SQdzXcH/kpTdfSCv2+/z555/d/Q4XJoSXYWtBYXChxNUOBZ1EBIoR4LdC/DMvuJuRli8ofqIuRfzt45KFe3MpYVXeu+++27ljlcrLca+kQVHC6r6lZOqpp3a/fVxFSwnxMFk1OInyypdVbt8JbXDEEUcY3LI7Q6x1ksGNlhWRvRS6rqyiTD6ULtxvUDAluaa+3Eb75jdMH4899tjEMVQ7s4+MaV999dXObILqFgERqDOB4qPdOjdG1XUOAWaJSr34lNOyqDUTyitWukDxxMNQkoyAfNiTccp6Ln7/zJazkqKXjTfe2L0YTD/99D4p0TcvLEicEmSqqaYyX3/9daJyopni2hc9XmybwM78bVsXt2LZSh5jNhalAvcHlEWVSDVtYRVHVtkkaCoByWsh11xzjXvhpJ5SQjuwfOKFtpGk1O8TRZJ184ztEhYcCNYKWELxIrnOOuu4l0YsUnjRlIhAMQL8hrAa9TLFFFMYFLbF5OOPP3Zxz3hhT3I/jrv3FiufmHwspLL44osXyxYew0IVRXUpwWKSVcRQXkWtWkudFz1equ8sNoFiLPrsip5fj22uKfGmeIZ6KXRdUXJjDcukC9byH374obGujf60pvrGwhVrNyZtmbxtBGFMy0TDTz/91AjNVRtFQARSICAlUwoQG70IbvpJXDjK6SfWUVgGsNIFL0sosiTJCTBzSFBSmRcnZ5a1nAwAr776asOKWLw4eyF4MQE6USSUIz74My5E+cLLlX/Jzz9WaL9Q+wrlr0U69x2WXSagq7diqUU9xcrcc889XQBhG0+qWLaKj7GwAkFZGWSXeunB5ee+++4zKKUmmGCCiuvsjBNL/T5pU6EA+F7JhBUCL7YEJyZYuo2jYnjZLcdaozP6rjo7nwD3Rf8bpDVsEyy6kGAZuswyy7iVOiu1oCxUNuk2QoW7t9WibKwcUawTAB9rWQLuo1RJ+gJfqu+UhQsW90bKR4GDMr6YNWIxFpUey7+mlFPouto4eI7JwQcf7FY8xoIetzrcuZtJeG7zHEHRRPiJRhFNnDbKlVI7RSA9At3SK0olNSqBSlaTK9VXFEusssTLAS/UkvIJ8FCWeXH53Dr7DBQnvAQwyGV1sDPPPDOVJmGthLA6W77gcpZUKVGr9uW3Kck+qwGxkhuWBGlaUyapmzw33nijsYGmnUKDl5S05YYbbnAulMRoYeU8XgoLCdZcKFmIrcHqao0mpX6f9KfQb9RbKhAbhxhM3jpuqaWWcqtzoQDkpdfnazQ2am/tCcwyyyzuXsJvBuG+0jPiphxtwZNPPuni95xzzjlmyy23jB5KbdsG8Dbff/+9qye1QscV9MUXXxgskexiBy4FxQvPHCb2sJQtJkn6jssvf89YESIor3ju7LLLLoYJxHoJE23cE7DS9QpDrisrTuYLrsjk9TLddNMZrCdhk8RV0Z+X5W+sfZmw5XnJBG4jiZ84ZUyLC7REBESg+Ql0qiUTDys+uFLxYVtSXwK1YM7LIgomBjsrr7xyxUuB15dE9mpT8O/sXZNSLSL2DoGLUTAx6CfwbKUuYPl14QaAxMXpIi3qKpJ/rt+vZft8HeV8E7vIrhDlXD/KOS+NvLyQYGFEHA8UGyjD+RBzBUGxUU3gbX4DvMDa1Z8McV9KWa4RXwOLBCzSfFtw/0DsqkouLe7auwwZ+K/U75Mm4voSJ8QUQwjc6xVM7PMsIY0XRWKeSUSgEAEU+sS7w1oOyzgsJIl9hxDbB8s4hLhhLG+Pggn3WGLm8eHeiKC4IU+58vzzzzuLH3/eG2+84Vzl8hXLlO2VQz5vku9oH+zKeObdd98NP9z77UqmoYKpUB9K9d33YY011gjLph4CnaPY8ePFYn1gEoPxHwoeL8SPQjHmhYUoKNdL/r5PxxWQv3+7aqVzmaUc7qW0DyEOk13Fz23jisx9kvsEVmRYQC6yyCKGOFfNICjZUTDhXt5oCibPX8G/PQl9i0BrEKi7komHFGa+zEiuuOKK7oP5J3F82CedD0onSWMS4PrxMnHXXXe5AIzMXtulhd2MPi9edonZxGbdjUkgnVZjycRsZbluUOnUrlLKJUC8GV5ebr31VnPuuec6JQV/C2mJfxHHlSEqzDDjzkDsj2JS6/YVqzvuGO3hHgEzbwUTl69WabwMsQoRL0O4qfkPLzIIKwBi6VSJ8DwjXhXxTHD9KqRciZaNgomYRMRx8W1B+YXwgkwaAYqzKrzoEk8m//dJe0kjKDMKxTjht83fin/Rj+ZBwcR1KjceTrQMbTc/ARYPwNoFq0FiILGqIe6WyCOPPBIqnLg38zePUgp3LP8hiDbCJAF/s+UKSg6voOZclCi9e/fuUAxlR1dQ7JChQEK0DwWyhMmF+lCq7/l9CAvM2yjWB/6GmVz0yh9O3WGHHVyQcl8MFu7RVYbZLqTQJ533hp7WKo2JS+IIYrWEsO1XVSPYN25kKJZwv+beffPNN/sqG/obJR/eAShGWeGwUUXW+Y165dRuEaiQgNX4102s/3Bgm5n4Y5e8r1vbWrmicq5J0rxWkeSQvvnmm4GNSxPYWfrADvoC+zIZXn8b3DWwg4bAPjwDOzgL7GpbrXwZOvTdzvw5VjZGS4djSqgNAX7fdrawosI33XTTwFotBVbJVNH50ZP+/PNPd+35u/FilSKBfYFyf0c+jW/r7uXyWnesaHKH7TTbl1/4J5984tpgrX/yDxXcf+2119w51vS/YJ5KDlTSlmg9VhHu2mVXd4wmJ94+//zz3flWyZT4nEIZX3/9dVfWxRdfXChLptLtalSBVZQF1n0zbJe1LAisRVJgY6WEaXEb6667bmCVBIF1MQoP24DfgVVeBX369AnT2LAxzoIFFlggJ63WOzzTuD/4Z1ut61P5xQnw92mVlh0yWdeugPtnpXLVVVcFDz30UKWnlzyP+7hVPpfMV02GRuuDdYELeD8oJtaSK4BdKbFKroC8+WIVjoG1DM1Pzvw+903uO9ZKL/NtLdVAa23n+mInTktl1XEREIEmIFA3SyY7SHQmr/ZmmVjsYN3NbjKLIakdgVrG/6BsZvOZcSIALqu9EHsEaw9cSVg1CJcZVhJidgoffEyhcRchngnm0K0qM800kyHOhIJ/Z/8XYF9KXJBRVsUi5gAzj9EPbhxesNJkRi/O4sPnifvG0gMzeVwRcK0iCCvuAQRn3WyzzXICZxO7gTqIrYGU0z7azblW6RvXjLLT8tviC8BFAomb7fd56tUWX1+x7yRtweqQYKzEdyIOSPQ34Ld9IFpcXuB8/fXXF6s29hjXnXO99UVspnGJBFSPC6peKH3VVVftkD9pfbiIEh/MKozcfQu3HbaxQsDdw0tc31kliecDzwKsNvh9sIQ6lkzRFUt9GfoWAaz/GCcSp8cLFnXVxODhNxhdqMGXm9Y3lpJY9tRSGq0PuMDhulxMiLGUxDIYq0fyerHvae43glV4ownPe6y8LrzwQmMn3Rut+R3ayzMLUazRDmiUIAJNSaBbPXrFS5C1aKm4KtzoeFBIakMAdxVMm3EfSUtWWGGFgkVhyszHm7KTkZczO2vvPsQywOzcm1PjgoEJNJ+FF1443GYw0eyiuEyNcYW9ogDFD/Ej8gV3Ub/KEC9EKA7tbHt+tpL7LG9PEFYUVcRm4IWKl3KUuFHxdfgg4eW0z1oCVdy+aBv8dn5bfLpXMlmLFJ/U4btebelQcUxCkrYQ3Jv4Lkgh9w9WP+IlmPhK/A4qcYcloDDnFlqtLdr8Qs/eYun5QdiT1jfvvPO6AOfbbbedU4IRjwbXGV6SorGW4vqOixNBiQcMGBAG9kVpa60ywhgs0X5pu7UJEGsHhS5/P0xWpSW4udZSCBdRa2m0PuC6WEvhN8I4t2/fvrWsJtWyuYaElyC2WK2Vkqk2vEhhfuIUJVPcxEeRU3VIBESgAQl0wRqrlu3mZajaWUgGvAxAfayMWra3FctO4xpFuaFgsi4F0aSKtnlB9oon/40CigElM1pRhZPf5qW7mQQ/fAYZ0ZnaZupf1vrC74rgmvxNdKZg7ULQUyyW8hVItIsZfCwCUdbmB5ZNo93ENWEgSIygzha1Jf4KYOWDop7gxfWQcutDKUe8m0ruybwUooT1sVfy+8czBsUXy6vXS7CoZsKLZxv1S0RABESgFgT23ntvM2TIEHPFFVc4pXst6uisMtdff323GAreDBIREIHmJlBzU5BqFUzgx8KG5WAZ5Glwl/4PkhdqAsqmtdR6GgomesnMNysz8YkKM/Be6cQ3q6pg3YEQXNYrnLzlkw+YHC2jUbYxL8bNhMDOfvWmRmm72lk7AljxzTPPPDWpgICp/M1kQcGktsRfYpQrBANnKet6SCX1sWpfpRJ1d6m0DJ0nAiIgAo1GAItnXEBtLFMXaqLR2l+qvYxpL7vsslLZdFwERKAJCIxXyz6kbQ3QqMt21pJxWmXHWUtUUnZaCqZidbOKFqugoBQjPg2uF8S3YWZk8803N1iBsHzt2muv7WIaoZxhtp+VR3hpxaWpUUQ+7I1ypWrTTlY4Y+nom266qTYVxJRKHLDoKkkxWeqWpLbEoybm0z333OMsheJzpJta7/oKtZ6VaPl7wKJVIgIiIALNRGCXXXZxCqZrr722KRVMXCvGtFjmf/XVV8106dQXERCBGAI1tWTCwiRNUbC4NGl2LAs3oWoszzi/syzN5pxzTsMnGufJrjASxnjyFk8+TgpWGt7Sycd74tuuDtYRTCemTD/99AaLAOKvEHtH0hoE+B0Sy84L8UfqJfmWg/WqN64etSWOinH3hPgjtUmtxiopzRYR04O4T3xYJEIiAiIgAs1AAHdkLHyYCLUrwTZDl2L7EJ04ZSJYIgIi0LwEahqTKclKEOWiVTyEcomVlx+XRPzB33rrrbJOzEIcmyQNZuUj72rHbLjftsveGmJ/5bvaoXiacsopkxRdszwbb7yxi0N111131awOFdxGICsxmXQ9REAEskVAMZmydT3UGhFoFgLbb7+9ufLKKw2WyxtttFGzdKtgP4glOcAu8BBdcbRgZh0QARFoWAI1s2RiQCZpPAJYIhHzCFfHJFZNLKsaXZ496z3G7WOZZZZxn2hbowonFE933HFHTpynqLUT2/WMj8TMz7nnnhttrrZFQAREQAREQAREQAQamAChH4YOHWpuv/12Q1DsVhCtmtwKV1l9FAFjaqZkqgVcVlFCedVZLlm16FNWy0TJ5Dnfe++9Lv7He++9Z1DSMOuC+w7HfZ6s9iNpu7Bg4sPS216I8xRVPl1wwQXmiy++cIdRMkWtnlA8sQpWLQQlE4G/P/vss4KrLdWiXpUpAiIgAiIgAiIgAiKQPoEtt9zSXH/99S4GYiu5jjGmbaTJ6fSvvEoUgdYg0FBKJlyaJPUj4BVIfI8aNcr88ccfTtFBnKAjjjiifg3ppJp69epl+ETNl7/++uucOE/MPp100kmuhVNMMUVsnCfc8KoR78NOXKZCS3pXU77OFQEREAEREAEREAERqA+BzTbbzC1Wc99995k11lijPpVmpBbGtCNHjjSff/65W5wnI81SM0RABFImUN3bb5HGeAVFkSwVHapVuRU1pkVOwnrn0ksvNQcddJA55ZRTzKBBg5xpb4t0P6ebM8wwg1l99dXNwQcf7Gag3n33XfPrr7+ap59+2rDyEdZMzz//vGGVEB6kWN/xvcMOO5ghQ4aYp556yvzyyy85ZZbawWpsrrnmMoUC32Pd5z+lytLx2hJgVnK55ZZzykBmJq+44oqKKvz7779Nz549ww+rJXrBuu7AAw808803n1lppZXM4MGD/aHE3yiNsb6rdjYxS22phj3KYlYtI8A1s8s33HBDYpZxGV9++WW3CAH3A/52WZDgtddei8vaIW3//fd3Qf4J9B/9nHjiiR3yZjEBq8tTTz3VLLnkku7eN3DgQEMsvHJlq622MltssUXOaXvttVf4N7HqqqvmHNOOCIiACDQCAeJsEpKhFRVMXB8/cVpoTNsI11BtFAERKE2gppZMffr0MQy20xQpmdKkmays888/30w77bSG+EsERn777bed0qR3797uBSpZKc2ba7LJJjPLLrus+/heBkGQ42rHy/htt93mLMLIw8tnfpynYqsl5fuwo1RC0fXSSy/5KnO+UQhOPPHELrZWzgHt1IwAigoUFEsttZTZY489nOKPFWOGDx8eWrslrZzfD8v8rrjiii5OA/dShDR+a/PPP7/ZaaedXID+fffd13z88ceJlU1//fWX2Xzzzd3v888//0zapA75stSWatijOD/kkENM37593XV76KGHnHID91T+jsqVxx57zCmsUA6jKOFaXnfddYZr+Mgjj5hizzDyotCfZJJJOqye9uOPP5bblLrnHzt2rEG5ysqe/A389ttvTrn+3HPPGbhwT0oixAOEWb4LyVprreWeOSjsUWZJREAERKCRCBB36eGHH3YKppVXXrmRmp5aW/3EKdb5rRKHKjV4KkgEGomAHdTWTOxKcIFlkdpnv/32q1lbVXA8gdGjRwd2dbXArh4XZrBui4GdiQisxUaYpo1kBKzCIbj55puDww8/PLAvTMHMM88c/n3Y5bmDNddcMzjssMOCm266KSCvF2sZEEw33XQBf1OLLrpoeE6pvy9rFeGL0HcCAvCM/tYTnOKy2JfrwFoWBVZRkXOKtWAJunXrFvz000856aV2rPLHXeP862etNwK7MkvA36UXa1kYWOVv8P777/ukgt/2Zd+1k/z09fTTTy+Yt9SBrLSlGvYjRowIJpxwwmDBBRcMuK95OeusswLr5hpYpZBPSvxtle+BjV0XWIvF8Jxvv/3W1WOVxWFa3IaNA+eui7Vcizuc+TRrbeV+izZ+X9jWRx991PXpkksuCdMKbVir2cAqllx+fqNsx0m/fv2CBRZYIO5QzdL8eIZviQiIgAiUS4D72eSTTx7YScJyT226/NZd0I13m65j6pAIiEBIoGbucvYFxs3YYu2Slqy77rppFaVyEhIg2DXuXVgxebEvzeayyy5zVmq77babT9Z3AgK4zmAqffzxx7tg6vikMyOP2fSee+5p7ADE+elvuummztqpe/fu7u8IFzz7ouosW5K63dCcM88801mfEchdUjsCWG7gRnncccflVGKVhgbrDqz/vLzyyituqeJy3SaxiMLKZptttjETTTSRL87suuuu7hrzN1lMnn32WWcFhSUTrmXVSJbaUg77/D7DBB5bb7214b7mBYs0rDbLdXfEcgfrMNy8+Fv2giXoaqut5izP/v33X5/c4dv/bXt3gg4ZMp6AhZFVAJl55pknbCkunfPOO6+xSqYwLW6D64CFnlXiOO6KPxdHSWkiIAKNRuCff/4xjAWeeeYZN9bjHtnqwjNO7nKt/itQ/5udQPuoukY9ZdCJy0e1cuSRRxZ1M6i2fJ0fTwBXORRM1oomJwMrq/FSizuInVF2CpKcDNpJTAA3OT7R4I/EeXr99dfDIOP33HNP4vLiMuJ+soIN4M5Hkj4BVhtEoRcVlAnEXWBFxsUXXzw8RBykK6+80ikcylHCo5xCcOuKCnVbizhjLZmiyR22u3btaqzlkosXhsKyGslSW8phn99nv1pk/t8F9zvK9f3MP6/QPq6zH3zwQYfDLFrxwgsvuFhdxRYC4G9+ggkmcKt3ouDHRW6hhRZyLyhcvywLyvKvvvrKKUHz27nEEkuYO++8Mz85Z99OfbmVS3FRhL0U4zl4tCMCIlAjAoQfQPh+8skn3bZXBFV7H0J5zgQ5rmFMJuJOL2mLy8QE0SeffOJiIYqJCIhA8xGouZKJwbt1PzG85FYi1pXB8LJGzBFezPBhnmWWWSopSueUSWDo0KHGum8UfDlgth8LDYKxomhKQ5lYZhObNjtWEASQ5oNgVVGtEFCYgY6ktgRQXGCpxoDSurWZe++913Af84IlG5YerNJYjnjF0DTTTNPhtKmmmsqw8mExIRAznzQkS22J9qcU+2hetq3roUviGqEI8fLWW2+5lW/ylev+eLnf11xzjbNEJE5RMUHJxN86Fj0sGoBlFM8/rhtxp1h4IKtS6jfx888/u7+HQnGZsM47++yzs9o9tUsERKDJCKBU4t2EbwTLcRbFQLgXs+0n6LCwLFdYkZnFGxgn5z9jyi2r2fJ7a12smVhwQyICItB8BMarR5eYCUDRVImwhDxWTLw8bL/99m4mmIDJrFjDSxyzBJLaEMCKCbctXngKCSuqMUvD6mn+JaNQXqVXRgAlXhqCK061s3JptKPZy3jnnXfcAPX77793XWWmDisNLwQvJtA0LlTlCAoHpEePHh1OY3Bcz7+/LLUlCqMU+2hetnHlIggpVpk2dpA7TMBvPymCkrBaoVzcihlUH3rooUWLY7VKlDAolGwcLzfLy28FKyhWrMyylPpN0Pbvvvsuy11Q20RABFqEAGMhJka9goluewVT/jZ5UP6XM37CGh0XORufzr2rRCcxWgRx0W7aWK9uhVxNfBbFpIMi0NAE6qJkghA356QzAX6mE8XUsGHDzBFHHOHO5aaN+8nyyy9vHnjgAcPLGqvwsJQxKwSV69rQ0Feuxo2HLy820VhMharkBQ0XEBRNknQJ8Hdz7rnnplaof3lOrUAV1IEA9yP+dnAdwtoP5fiFF17YIV+5CVgrIcyO5gtLxONmVS/JUluifS6XPf1gNTcUOqussoqZbbbZnHUT++utt56ZYooposWXvX3DDTe45xSrSd5///3OOqlYIViOMqFC/CbuqcQlOumkk5zl21133VVXRWKxdsYdK/Wb4Jx6/kbj2qg0ERABEUC5VMlYiHMYk5USlFUomPDAYDKc1YElHQnkr5rcMYdSREAEGplA3ZRMQMJ1jhl9lEfRGCVRgHblLBdzBIVU/s2ceBcM/M855xxnfsoN3K7C42aizzjjDLdENO4ExAm6/PLL3XLf0bK1nZwAVkzM8nu/9GJn4r6Doom4Qcy6S7JNIDpzl+2WNnbrmKljUIrSHOV4tUKcGiRuKXvSsGaql2SpLXF9Loc9Syi/8cYbxq6MZuxqgMau/uheDGBarrVZtC24fqFkXHrppc3TTz+dqCxmy4nhlS9YlCKl4m7ln1fP/VK/CdrCdZGIgAiIQGcR4L2imjFQKUUTFswomEaOHOmeI8QvlcQTUPDveC5KFYFmIVBXJZOHxk3+5ZdfdgonlE4olPiwjekk2yikSgl+vDvuuKNbKQlXEawH9t57bxebBKuanj17uqCp++23n1OApOH6UKpNzXAci7C7777buXgk7Q/KKAIaY1FG/BFJOgSwapBknwCuiChYsUSJCoofgrrHBYOO5kuyTXBvJL8OLJsIukyA6HpJltpSLXvcvIjzhyvbWWedZTbYYAP3LHrzzTdz4jSVw5YXEZ47rASIVWgS5QovJ4MHDw7d9qL1MaGClBvHK1pGrbf5rTMRlP/7pF7SWFnTWynXui0qXwREQATyCaBc4t5crVBGnKKKQNYomHALJgZTOQt7VNumRjwfJdMPP/yQyvioEfuvNotAsxPoFCVTPlQUSkmUSvnn5e+z6tJhhx3mBum4j+BeQKBwYmIQfA/XOvaZsX7xxRfzT9f+OAKsarTgggsaAhSXI7jW8UHB55fiLud85e1IAAuxtIW/EUm6BH755RenYOXeEhVWqkFBUMhyM5q31DYDMtyurr/++pysrNqFAh0rnHpJltpSDXsGuFi/Hn744TnorrvuOoN79rbbbpuTnmSH+ycTKSiZrrrqqsQuYihgWFltp512Mix57YVryzVm0mTOOef0yZn83nzzzd1zN+rSiaIcS656/j4zCUeNEgER6FQCaSiYfAfyy2KiBwUTzyNc5Oabbz6fVd8FCDCOQAj+LREBEWhCAtZ6qCVkxIgRgX1hD6z7QmBng4nCG1hXiMAOioNLLrkksC+CLcGhVCdtkGLHxr4olcpa8Li1agqs22Ngg7IXzKMDyQjwO03706dPn2SVt2AuWFt33rJ7Pnbs2MAqygO7IlhgB5+Bdb8KrAtvYBUDgXV/CoYPHx6WSfk2FkFgrZvCtPyNP//80113uyJgzqGLL77YpVsFRmADiga33nprYGPhBJtttllOPtpAHdF6oxn83/npp58eTXbbVgnpzv300087HIsmZKUt5bCP43LAAQcE1gInsLGZHFPrKhzYlc4Cq3iKdjdIwsXOZAfWosddc2vZ5s7hvOiHa4tY92LH2Sq0wnpoC79BnlPPPvts8NJLLwU29mBg4zO5a+0zct25vrfddptPKvht4zsFfPKlUHr//v075E9anw1c7tpqJ3MC++Lg+sCzwK7iF9iJn7AJcX0PD47bsPGxgrXXXjs/2e3zjLErmsYeq1WitbB214ZviQiIQGMR8H+/aY6n/L3ALhYRWLe4wC6SE3z00UeNBaaTW2sntIMDDzywk1uh6kVABGpBoJu94baEEND1v//9r/vQYdz1sHB67LHH3Ko9LBNtB63O0glrJ+IRYfrfasIsPO49u+66a8Vdx/oGqzIsmoYOHVpxOTqxNgSWWmqp2hTcwqV27drVxfLZZ599XMw54s4hWDDdcsstzlXI47HKG+cWbJUNPinxN1YuBKXGUgbXLlyUsNIkJl1UfB1Ri5Lo8WLbrIaH23Kp9mWlLZWwj3LBeojlqnG9RqxCxOy5556OcZRTEi4E9/YrFJ188snR08Ptgw8+2Ew44YQuthaco6sCEuSbY1xPrKkQnl3MjBMM3AuudZybZLU23P7ipFg6QcejkrS+eeed1wU432677dxqelbp6p6pBL7HktgL8a7y++6P6VsEREAE0iYQ596WRh08F3gG8xziPs39WpKcgIJ/J2elnCLQcARqoblqtDKtO0Jg/acDrAaYjbAX0X2wTDjuuOOC5557rtG6VFF7bWwSN6N/7LHHVnR+9CQb6NgxtC9N0WRtl0lgkUUWCX+P/ndZ7feRRx5ZZitaJztsK7FkihKy5vKBVVoE9sU8mlzWdiFLJl/ImDFjnNXN33//7ZNS/bbxcwL6kUSy1JZq2GOFVMjyy3Moh4s/p5JvritWQXaFwoKnWyW+e24VzJDygXLrwzrYKkRTbkVbcbJkqglWFSoCTUuA53q1Y6f88+1kamAV685y37oFNy27WnbMLuQU2JiFtaxCZYuACHQSgZaxZLIPh4JiXSOcLzX+1Mjnn3/urJywdGKVtUGDBrkV7LBw8p+sx8bwnfWzN3xjtYWFFoIlRL7QV6wXiKtUrbAK4AknnOCC6WIhxkyPpHwC1lXGWVmUf2bhM1i+V1I7ApNPPrmp9Yoy3bp1c8va16IXN954oyGwN/1IIllqSzXsp5tuOsOnkJTLpVA5SdKxAMIqqJAMGzbMPPPMM+a0004rlCXV9ErqY2EOiQiIgAhkgQBx7dIWYglxn2M165lmmint4luiPOIy/fzzz8ZOqiiOVUtccXWylQh0QbnVSh2upK88SFDQoHTiY2OAuJuhd6vje4oppqik6JqdgxIpPzBhXGV2didUOOEmgjKIFY7Skq222so8/PDDLtC6XjrKp4pyMG2lkP7kC18HlpCP/k0UzlnbIzaemUH5Peuss7pg37im+WXsa1uzMdZy0ynJbCypWldVsny1JR4RLhoEB+/Vq1d8hpRT611foeYzccE9kRVQealD+VUv8fdiG4fFWCvnelWrekRABFIgwLO9lvKf//zHWMtz9+zkmw8LdUiKE/BjHVal5n1BIgIi0DwEZMmU4Fqiaedjg9MZ6xoSKptQOJ177rmuhOWXXz60clpmmWUSlFqbLAyErdtf4tXdUEQRHwO/aF4k0rBiivaM+ExLLrmki8+Eok5SHgFeZmzg3MTXs1TpWOVJsk+A+A7EBfIy9dRT+82afy+99NI1ryNpBWpLPKl6K+zrXV98r41TLGHhxYfYgRIREAERSEKAsRTj4zSFMlntlZh+fOyCH+b22283xNZDmHz2Ciesm/12fsy7NNvUaGURg5AxLpP5UjI12tVTe0WgOAFZMhXnU/LoyJEjndLJWzqxXHOPHj1ChRNWTjaOR8ly0siQ1HqpUF209ZFHHil0uOJ0Hh4omggmPmTIkIrLadUT/Qx6Gv2XFVNxilmxZCreSh0VARGoNwF/H5YlU73Jqz4RqJ4A42Os9P3CDNWWyKIbLPRBufli45uGiqeoAgovCCaQogonv501b4j8PtVyn0U37Gq75sknn6xlNSpbBESgzgRkyVQlcEz2t912W/ehKB4o3q2OB5AN4OpipxALCSUOHx5OtZAk7nHF6h0xYkSxwxUfwwrMLg9uBgwY4Fbwq2bluoob0cAnMluGC1e115eXI4kIiIAIiIAIiIAItBIBxlHVjqGivIopq1iZetlll3Wf6Dle4eS/b731VhePiDzzzDNPB+UTLnitILwj3HDDDa3QVfVRBFqKgJRMKV9ubw47cOBAFzPDK5ywdLrgggtcbTx8vMJpueWWS6UFUdeaSgv86KOPzCGHHGIKLb1dabmcx5LWb7/9ttltt92coimtflfTpkY618+WVTpIQknFIEsiAiIgAiIgAiIgAiJQHQE/Lktain8/iObHggc3O694Ou+88wweEgiT2JzjrZ3YrpdnRLSNtd5GyfT777+bt956yyy44IK1rk7li4AI1ImA3OXqBJpq7HLUOQHEP/vsM2OX7gwVTlg7FVtRqFBTedBVqnyIK7OW7gCslsZD9cUXX3RuhXH1K60wgXKvNYOTs88+WwqmwkhzjshdLgeHdkRABMYRkLucfgoi0NgEyh0/FettLRcI4V3BK528Aur99993zeGdwSurosqnWgc2L8ai2mMsYjHBBBO4VfqYkJaIgAg0BwEpmTrxOr755puhax2WTqNHj3arBXkrJ76nmmqqki3s27eveemll0rmS5KBBxWBy59++ukk2cvO8/XXX7v4TAT6I0CipDICpQZL8F133XUN+STJCUjJlJyVcopAKxGQkqmVrrb62qwEWK0XBU4xd7dSfa+lgqlQ3b/++muoeIoqoFDQdOvWLcfaySufJp988kLFZS69T58+ZqmlljLnnHNO5tqmBomACFRGQEqmyrilfhYBmXGt8wHEvdKI1ZW80qlfv36x9aY9gzHffPOZd955J7auNBLp5yqrrGIOP/xwc/zxx6dRZEuXwcsPH+8O579bGkqFnZeSqUJwOk0EmpyAlExNfoHVvZYg4P+OK+ks8VRRTmVlAZV///03x9XOK59++eUX1z08I7zCybvdZXVVTmK14i737LPPVnJpdI4IiEAGCSgmU0YuCi+3KF74IN98802ocBo6dKg57rjjDLMSKJx8EPH555/fKRfS7sK7776bdpE55dEHVvkgMHrv3r3N5ptvnnNcO+URQKkkxVJ5zJRbBERABERABESgtQgwVkJJhJV3uWEmUNRgxZQVGW+88QxW63yiMnz48BzlE6s6f/nlly7LzDPP3CHOU69evaKnd8o2cZmuvPLKTqlblYqACNSGgJRMteFadanTTz+92WKLLdyHwoYNGxZaOh166KEuSN4cc8xRs5XqopYxVXcmpoC9997b9WmHHXZwgcAV7C8GkpJEQAREQAREQAREQARSJeBDCZRSNHnrpc5wkau0w3PP/f/tnQWYJNXVhi8srgm+C4uzOAQLLK6Lu21wWyS4BLeFLBLcJXhIcFk8aHDdJbgHCe4Oi9V/3vNzK9U1LdUz3bPdM995npkuuXXvrbeqq6u+OufcAYG/DTfcMK0CkSl6OpHn6aqrrgrDhg3z9exjuTxP6cbdMIHINGrUqPDkk092EM26oXk1IQIi0AQCCpdrAtTuqJKwOv4uvvji8Omnn4Zvv/22oc121w8q+Z9+/PFHTwTe6LC/hgJRZb2CgMLlesVh1k6KQN0EYphNMwfGqLtT2kAERKDLBKLgNHz4cBdiqBDRBfGFNBV4P/HX04ywuig8xU8EKMLwxh577A6hdjCZaKKJmoZh3HHHDWeeeWbg5bNMBESg/QlIZGrzYxhvfBu9G911I82IGSQuX3fddcOFF17Y6N1QfSJQFwGJTHXhUmER6DUE4m9td/029hqw2lEREIGWIUAi8TiiXRSe+CTxOEbO1nyep2mmmaYh/SfxN6F/CE0yERCB9iegcLk2P4bNervSrHrzuGefffZw/vnnhw022MDD5vbZZ598Ec2LgAiIgAiIgAiIgAiIgAg0kUCfPn3Cggsu6H/ZZnghjNgUBSjyqr733ntepH///h3yPM0yyyzZzQtNEzL3+OOPVy3b7FQeVRvXShEQgboISGSqC1drFp577rnDc88917DO4RLbnbb++ut7AsY//elPngh8lVVW6c7m1ZYIiIAIiIAIiIAIiIAIiEAZArwQ5m/jjTdO177zzjtpuB3i0xVXXJGOGP3b3/62Q54nPKCqGSLT2Wef7eF6JDVHUOLv3nvv9VH9ELmi8SI8jrgdwx3jOn2KgAi0BgGJTK1xHLrUiymnnDIQy0zSvEbY2muv3Yhq6qrj0EMPdaFsm2228fxMvBmRiYAIiIAIiIAIiIAIiIAItBaBaaedNvC3+uqrpx374osvUuEJUYjw4pNPPtlH9BtnnHE6hNrxUnvCCSf07RGZCNcbOXJkuOmmm/zlc0y8njbw60QUoOJyCU2RhD5FoHUIKCdT6xyLLvWkkUmzGd51dNjXX38dFltssdCvX79w++23j44uqM1eTkA5mXr5CaDdF4EKBHioWW655fyhqbvCySt0RYtFQAREoG0I/PTTT2mYHcJT/OOeH5trrrlS8enggw8OM8wwQ3j11Vfr3r/uGrCo7o5pAxHopQTkydRDDjxvC7gB7qpxkR5dxqgV5GdCaNpzzz3DSSedNLq6onZFQAREQAREQAREQAREQAS6QGCsscYKeCnxl7UXX3yxJM8T9/yMNv3uu+9mixWeHjp0aOAFgF4CFEamgiLQVAJjNrV2Vd5tBLioLr300l1qrxXeAjDS3HnnnefutXzKREAEREAEREAEREAEREAEeg6BOeaYIwwePDgcffTR4dZbbw077LCD79y3337b6Z1sxMv2TjeuDUVABEoISGQqwdHeMyTHI7dRZ6wVBKbY72233dY9mYYMGRIeeeSRuFifIiACItAlAtdee62/TeUza3H5ddddl10crr76ai+fX15SSDMiIAIiIAIiIAKdJkBOJTyRGmHNEpp++eUXzxnFZ9by89l1+ekffvjBE5vnl1ebJ4VJ0TQm9fQl22bR7YqWy9adnS6yfdF9zdbbjOkifaXdfDn6T24x/nq7SWTqYWcAF+l6Qt4Y7YFQu1ZLmnfiiSeGQYMGBRKBf/XVVz3sKGl3REAERgeBjz/+2JOK8pm1uPyjjz7KLg6VlpcU0owIiIAIiIAIiECnCTRKYKID+aTgne5UbsPNN988MGpeTHT+9NNPhxVWWCFMNtlkgSiM1157LbdF6eyHH34YZp111sB2RQ3HgZlnnjlMM8004cgjj6y4GXlsiWaZeuqpfdS9N998s2LZ7Iqi+0BCd7zOSMQ+55xzhlNOOSVbTaHpP/7xj2HDDTesWPa2224LjC7et2/fsMYaa4Q777yzYtlmrrjyyis9R9gUU0zhoylW8qzjeG+wwQaeRxgmsb/HH3+8Hy8G5er1ZoqbrIcSMLEp2WWXXRJzSSWTd/o399xzJ4svvnhi4lJL7/l///vfxEauSOxL3NL9VOd6DgG+J3xvZCIgAiKQJcDvJdeHVv/dzPZZ0yIgAiLQ6gTitTX7nNKVaRNCmnIft8kmmyTHHnus4zSPpMRe0iennXZaYiJEctBBByULLLBARdSPPfZYYgnO/TfkySefrFguu8LEDq/z9ddfTyxZemICVXL99ddni/j0N998k5gwk5jQlFiS9eSAAw5ITHDqUC6/oJ592H777ZOBAwcmb731lv/ZqIAJ/StqNlpgMvbYYyfrrbde2U2+//77xIS05Oabb07MMygx73HfJ/rYnfaf//zH+3H//fcnH3zwQWKCYrLHHnuU7YKNxJ4ccsghzvyWW25Jxh9//MSS2XtZE+8SEyTLbtebFsqTya5kPdXwTrILYHjhhRfc1dIu5P757LPPhgcffLDlk+NNN910ngickJVGvuXoqcdb+yUCIiACIiACIiACIiAC7UIAz6NG2ueffx5IH9JMGzlyZPjss8/CzjvvHExc8GiQN954I/B8lbd33nnHPXP23XffUI93ywUXXBDMUSDMOOOM7lmz4447hr/97W/56j3aw8SvsNJKK4U+ffoEcyIIL7/8cody+QX17IOJRGG33XYL/fv39z+iTC677LJ8lWXn8eDaa6+9fPuyBWwhXkGbbbZZWG211QKjPK+zzjoebsazancaXkxwXHLJJcNUU03lz57lmBMKh7fVn/70J2fOgFWMosixlv2PgESm/7Ho8VPtOOLCyiuvHE444QS/gCM2yURABERABERABERABERABHoGAcKwGmmNFq7yfUNQIvQNQQRjBL0ZZpghsDxvk08+uYsoW265ZX5V1Xnqmm222dIytFeufkLkCOUjrOuSSy4J5snk4ke6YYUJ6iq6D/369XMxJVZFW0WELMpvt9124eCDD3Y+cfv8p3l5heOOOy5dbJ5ELp4RhtadVo75J598EsxDqaQbiHns18QTTxzMiylsscUWLpANGDCgpFxvn5HI1NvPgDbYfxTwrbfeOpAQnCFPZSIgAiIgAiIgAiIgAiIgAu1NAK8jvI8abc0UmvBYyQtjk0wySUCQyNt4440XJppoovzimvP5NirVHyuysLpg4XQBzyHarGX5+ilfqQ1ELEYBfOihh8Jdd93lHlX53Jbl2jvnnHPCuOOO6yJYufXlllnIWth0001ddEJA607LM4EHVu64stxC35yFhUAGom/yYhRlerNJZOrNR7+N9h23UYY7xUVTJgIiIAIiIAIiIAIiIAIiIALlCDQzeoME4HlBgXkSdDfK8m3Uqt/y7QZGyh0xYkTYfffdwzPPPFO1K/n6KVypDV7yL7LIIp78m9C8ffbZp6pnEnW9+uqrPuI5IX+EEb777rvhyy+/DC+99BKry9oTTzwRllhiCffEIhSxuy3PBB54q1USu1hH2CCJ1klJc+GFF3Z3l1u6PYlMLX141LksAYQmRkIYMmRIdrGmRUAEREAEREAEREAEREAE2oyADbbS8B43U2Cis+Qmyo7gxjD2NlhRmNHyJzXK8m3QXrn6ifC4+OKL02bxqGFEOrxrqlm+/mr7wCjfjJpmib8DI9n9+OOPYZZZZqlWfXjllVd85D1ySTGqHALMo48+GnbYYYey2+HRxuhypEjZddddy5Zp9sI8E5gjMOU9w+Dxl7/8xTnQJ8LnyONUi3mz+99q9UtkarUjov5UJIBKf/7554fzzjuvU8NnVqxYK0RABERABERABERABERABNqewDLLLNPUfaB+G9UtXH755R4yddZZZ3m4FDmOsIcfftg9d2p14r333vMQtHLlCBk744wzwkcffeQJpXn2QazBbOSz8MADD/g0OZ9Iyh2TjpMjiNxCJADHCHGjnbzVsw+nnnpq2G+//byKL774wgeVwrsJq7QPq666qg88xeBT/NkIfJ5UO4YxZveBOmy0tkA7JP8mfJI/xCys0j6wjqThNvoek26EC5LTKdp3330X7r777mAj2PkiPrPzsRyfNoJgsJHt3AuM7RCSIvNRo0b5duSjIhcTeYJ5JsUQ36655pqWH1DLO9uN/yQydSNsNdV1AhtvvHGwISODDSkZ7rzzzq5XqBpEQAREQAREQAREQAREQAS6nUCzvY6asUNjjz22CwyEguE5dO655waiLfBowUgEfeONN9Zs+rbbbgt/+MMfypYjFy3hdySTXnjhhcPyyy8f1ltvPS/L808UPxixjpHEqQeRi7YZES0KXiynnbzVsw/kxiXMbfbZZw+LLrqot7HCCit4ldX2Id9mdj67D6effnpAvEJYI2Qt/iH4YJX2gXWIb4zcQQNPFQAAQABJREFUF+2+++7z0eniPGF69PX999/3RQhazJcT3khAvueee4bf//73nnQdweqoo47y7chBxXZ4rGEwv+KKK/z4wIUoG9jL/kdgDEtalfxvVlMi0B4E1l9//UDsLq6XjYyBbo+9Vy+bRYD4aly3Dz/88GY1UajeH374wX+4YmFGC4kuxk899VS49NJLA0PK9u3b19/+EH9fr3FTxI8nb5XKuWAXqa+zfeGN0AYbbFC2ie233z6sueaaZddVW9jZvlAnNwokqCSHADc3vAHkHKgUh1+tH/l13DThhl50uN/HH3/cE2zyVpKf53nnndeF9QUWWCBfdUvOc0PH+UluCN5C8gBxxBFHhAknnLCu/pbjhgt9vHHnxhu3/e4yvifLLbec511ox4ei7uKkdkRABESgXgL83g4dOrTezSqWb8ajLb9J888/f4mgwW874gPD3XfG6CejzjEqXCXDo2f88cf3BNqVysTlCCfct4w55v98SKibe8WVVlopFiv5rGcf6Mukk06ajqpHRUX2oaTBTszU2od6q+Q+85RTTnGu5bblHhVPpnxy93JlOf54NpHgPBr5sPAU+/TTT+OiXvn5v7OwV+6+drpdCeCiSIysEoG36xFUv6sR4EebWHDi6nmDRMJFjGXEffOmhrcmDJmLV1+9IhNvpHhbQ33RHblaf8qt60pfEFAQyXChfvvtt0v+cEGv17rSF25uBw8e7G+1GJKWIXMvuuii8Lvf/c7d1OvtS7Y8N83/+Mc/OiQIzZbJTuPCvdhii7kb/Oqrrx74wyWe44/I0er2008/hTXWWMNv3tZaa62w8sor+xveFVdc0W/Yiva/Ejd48H3gDSxilkwEREAERKD9CfA7zG9uI6wZOZ5ivwiDI0wqGmJOZwUm6uBFeQw7i3XmPxE6sgJGfn12HjEpKzCxjpHRqoUP1rMP9IWXsVkrsg/Z8p2ZrrUP9dRJMm/ONYS7SgbvIgIT208xxRQlx+exxx7zF5eV6u5Vy+1hRiYCbUnAHr7wwktslIO27L863XoEOJ/sBmW0d8xixv3ctgfqkr4MGjQoMeEpsTcs6XILH03sRz8x4ShdVm3CvKSShRZaKJlgggm8jZdffrla8YrrutIXc2/2tov2uWInfl3R2b5YToDEbrCSpZdeuqSJ4cOHl+VfUqjKjAlniYktXgfHhukiNs888yTm8ZPYCCxpcXPXTuyGJ1lwwQXTZa06YZ5xfi5aItK0izbcsXP461//mi6rNFGUm90wJ5ajr1I1TVluI8f4fvApEwEREAERaCyBeI3lPqyzf828f7OcO35/aF7Pjd1x1dajCNjLQj9PjjnmmB61X53ZGXky2ZVM1p4EGOby7LPP9hEPsiMrtOfeqNc9gQAeRg8++KDvCuFbJ554YrjhhhsC4W/RCPPkTRhDudZjJgZ5eNDmm29eMtIFI3fwZikmIKxVp92EeRJJPEI6a13tCyNwTDLJJB7z3tk+xO260pfnn38+9OvXL5D7IGt44/AWi2NYr+FmTVx/HM52+umnL1QFb9dISEnuAVyvo5FvAY8g3K9xa29lI0cBb0zJTxCNPBJzzDFHMJEpLir72VluZSvTQhEQAREQgbYiQBgy9yedNbbHI6pZRj4k6ifUSiYClQgQVs95EhOlVyrXG5ZLZOoNR7mH7iPhIzzMkKCNMBcS8PHFboewkh56SHr9bpH0j/AozkUetklIyIgZhAtFY7QQEjYyGkU9hjiFkXQxa4gkJJ4kBK6WMeIGo2UgSOHi21nral8YCYQcQ08//bQPV8uQtY888kinutOVvnBTShLHrbbaqqRtwvnIPdCZXFX2tsdFK8SvfL0ljeRmJppoIh/yNy/GEM4IG8SqvBt8rorROkv4GvkguB7njWW1zs/Ocsu3pXkREAEREIH2JMA9fGeEJrbhxY5MBESgdQhIZGqdY6GeFCSAiITnBmoxf8S/kguEh3dyebCM9fxYyUSguwgwCgW5gfCqY0hZhp3FU+bPf/6zD6dKUmmMhNdHH3103UmlqR8rJw5NNtlk6cgZXqjMPzynEMHI5cQQsV2xrvQFMQEPIYQm4uLhYyGvYeDAgZ5jrV5vna70pRIDRC+uIXiN1Wvkijv55JPdQ6rebcuVJ4k2+9jqo5bUOg6MHEMizUrWaG6V2tFyERABERCB1iXAvTv3CZXEpmyuHF4UIS7pfr91j6d61nsJSGTqvce+LfecJMeISEUMwSkfBlNkO5URgc4QYFQwjFArzj2SE2N4qGDRC4Xkxfvvv38gDKoeI5wKY/SzvHHTFR/y8+viPN5V9On444+Pizr92ZW+IMQhNpC03HL3hM8++8yFYhJFX3jhhZ40up6OdaUv5doZNmyYj/TCSGbVkmWW27bRyyyfUdhpp52C5dAKjDDYylbrONB3hFeZCIiACIiACNQiEMUmRCQEJ/4QlRjohGUIUXyyTCYCItB6BMZqvS6pRyLQkQDeS/ywEF5TjzFKFH/6IaqHmsp2hkAUmfbee++SzQm9wksDUaUrhrcS9u2333aohhHZxhlnnA7L44IrrrjCw/gYqazeoeRjHdnPrvSFEDREJjyW4ogpjJ525pln+ohzhIsx8l1R60pfsm3QH44dXkgMVUw+rdFpl19+uYfbDRgwINx6662paDk6+1St7VrHgW2rnaPV6tY6ERABERCB3kkAEUlCUu889trr9iYgkam9j1+v6T0PnfUKTFk4eD9JaMoS0XSjCRC2SX6khRdeuKRqxCfyD/Xp06dkeb0z1I19+umnHTZlWbkwOgr+/PPP7g0z00wzheuvv97/WB5zGeHZxDq8q4paZ/sS649eXnGez2mnnTaQzJ8hgnlDSbhaEetqX2iDvEebbbZZuPLKK11oOu644wq3X6SP9ZZB6CIxOzfW1113XZh00knrraLby9c6DnSoHfaj28GpQREQAREQAREQARHoYQTG7GH7o93pgQQOOuggz9/S1V0j74tMBJpFADEJkSQrjuB1hCcTnjpdNZJ7YzG3U6yPNki6PN9888VFJZ946Iw11liBnDgXXHBB+vfoo496OUa6w9OpHutsX2iDUdIQcfL7gbD0+uuvexhhlmGtfnWlL9SNVxXJ2W144nD66ad7OGE97dfqX73rCbVEVCcf1G233dY2wgwhm4SG5o8r+8+y2WabLYw//vj14lB5ERABERABERABERCBNiMgkanNDlhv6y5hckcddVRDdnvEiBFKDtgQkqokTwBx5OOPP+4gJo0cOdI9icqNuJWvo9Y8eXkInbrssstKig4fPtyFEobXLWd4DZGvKf9Hwm2MkcuefPLJcptWXNbZvlDhBx98EPbdd99wxBFHlNRPKN/bb79dMhJfSYEKM13pC1UyAtztt9/uXkw777xzhVa6Z/FZZ53l1yhEJhLIt1t42eDBg8MNN9xQEtLJMWVUw0rnZ/eQVSsiIAIiIAIiIAIiIALdRUAiU3eRVjudInDHHXekCZM7VUFuoxtvvDG3RLMi0HUCMR9T3mMpLs+KTCSzRBgp5/FRrSd41+CNd/fdd3soFUPCX3vttYGE3htvvHFYZZVV0s0RcGjjlVdeSZcVncBzkG3feuutipt0pS+EgDGSHGLZoYceGp577jn32Nluu+08qXlWfGp2X6K4NOusswZEaNrL/jFiZTRYw4XwtUbYzTff7PVF0RAR8MADD/ScWeSqyvYjTo8aNcqbrqcvnBfZcyP2vdLyQYMGdShftD1yWpEfjCTuCKwPPfSQT5OHi2MdLb/vcbk+RUAEREAEREAEREAE2p+AcjJ14RjiZYPxycMKYSnLL7+8v33mQYo/WdcIkEMGro0yHnxkItBoAuRjYvQ4RIissZzR4BAxojG6Gufh999/HxcV/hwyZIiPxoZQddJJJwVClNZcc81wwgknlNQR2yiXJLykYJkZvLKK9K+zfSF0D6Fmt912C0ceeaT/wQ4h7pZbbgkIEtGa3Zco8CDYlfOYJJdW9G7CUw0ujRohjTxa1BdHBSS59+eff+67fswxx0QEJZ/77befJ0uvpy+VctlVWx5HQoyNF21vjjnm8CTlW265pX8X8KJbYYUVwtlnnx0mmGCCWJ3nFcvue7pCEyIgAiIgAiIgAiIgAm1PYAzLg5G0/V6Mhh1Ycsklw4MPPugt81a/EkaG3OSBUNY5As3IjaIE4J07Fr1hK863VvjO4rHCiHQkf84LSByHn376Kbz22mth5plnbsqoY4Tl4dkz8cQT1zzsXekLw96/8cYbPvJepba6qy81d9QK4G1F2Ndqq61WpHhTy3R3X+ptD4EQgRUhtNHGCxyEL/KddZfxMkkDSHQXbbUjAiIgAiIgAiLQzgTGbOfOj46+c6PJgygjIEWrJDCxniSu9YzaFOvsrZ94d/DwfN9995WEV/RWHtpvEShHAG+g2WefvSkCE0nASaZdSfTJ96crfSFR9DzzzFOxre7sS36/8vMIGuSNItRvdFt396Uz7TFiYTMEptHNXu2LgAiIgAiIgAiIgAhUJ6Bwuep8StbikYRohNUTwnXssccG/qqJUSUN9dAZwkzeeecd/2M0rHLTvJ1uppHrBKGQN+EyEWh1Aoz8RlgToWkbbbRRt3S3f//+oVVyl7VSXyaccMJw0003uXdOtxyIKo10d1+6u71Kuz5s2DC/fj/11FNh2mmnrVRMy0VABERABERABERABEYjAYlMBeFnBaaCm5QUQ9zgDXjWA6qkQBvPENqDYJQVjvLzrMvmoCE/Bw8J8Q9vhjjdr18/nyYxMvk8GmkxcW4j61RdItBoAn369PGE3rHeySefPE42/XPxxRdvehtFG2ilvuCZ0yrW3X3p7vYqceY3grxP/PXt27dSMS0XAREQAREQAREQAREYjQSUk6kg/EblBmq3fECffPJJ6nGUF47ifExcG1FOPfXUqWDEQ0EUjaKIxCe5OopYo7hn2+rtHmVZFpouJdAqOZlKe6U5ERCB0U0AD1jlZBrdR0Hti4AIiIAIiIAItAMBeTIVOEoMG94o4ya1FUSOH3/8MRWPqnkgfffdd+mujz/++Kl4hHA055xzpvNZMYkRhRpljNZFAmKZCIiACIiACIiACIiACIiACIiACIhAaxOQyFTg+JQb3anAZhWLNDsn0GeffZYKSHgbxb+smPTBBx+U9G+qqaZKPY5IKLz88su7gJT1QurOkJ3YueOPP97fHsf5rnwyLPe+++7blSq0rQiIgAiIgAiIgAiIgAiIgAiIgAiIQAUCEpkqgImLEYQabQcccECncjP9/PPPqWAUQ9WigBQ/EZK++eabtMvkgsqGqeFJlRWOogcS5VrRSNA977zzhmeeeabL3SNZ+9FHH93lelSBCIiACIiACIiACIiACIiACIiACIhARwISmToyKVlCDqVG29dff92hyi+++KJEQIqiUVZMeu+990q2m2KKKVIBadZZZw3LLLNMOh+FpCmnnLJkm3acOfXUUxvizXTYYYe14+6rzyIgAiIgAiIgAiIgAiIgAiIgAiLQFgQkMtU4TF999VWNEvWvfvbZZ8MWW2xRIiplhadxxhmnJFRtqaWWKpmPnknjjTde/Y234RZ4MyEQDR06tNO9J0yOEQJlIiACIiACIiACIiACIiACIiACIiACzSEgkakG14cffrhGic6t/uijjwLDQi+xxBIdvI8YnU1WSiAKRJ0Rmg488MAwbNiw0go1JwIiIAIiIAIiIAIiIAIiIAIiIAIi0FACEplq4CSH0SOPPFKjVP2rb7311vo36uVbIDTxx4hzI0eOrElj7rnnDqeffnrAE0omAiIgAiIgAiIgAiIgAiIgAiIgAiLQXAJjNrf69q990KBBDd+J7bbbruF19qYKR4wY4eFz++23X5hnnnlKdn2uueYKiy22WCCXFmGJEphK8GhGBERABERABERABERABERABERABJpGQJ5MTUNbuWJyKsm6RiCGzx1zzDFeEaMASlDqGlNtLQIiIAIiIAIiIAIiIAIiIAIiIAJdISBPphr0EC4WXHDBGqXqWy0xpD5eRUqLaRFKKiMCIiACIiACIiACIiACIiACIiACzSMgkakA2xNOOKFAqWJF9t9//yBBpBgrlRIBERABERABERABERABERABERABEWgfAhKZChwrRCHy/3TVSER99NFHd7UabS8CIiACIiACIiACIiACIiACIiACIiACLUdAIlPBQ0Lun0UWWaRg6fLFGOlMJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAI9kYASf9dxVB977LGw1157hZNOOqmOrYKPgHbaaacpTK4uaiosAqOHwNChQ8O99947ehpXqyIgAiIgAiIgAiIgAiIgAiLQxgT62Chdh7dx/7u96yuvvLK3WeshdNxxxw0///xzOOCAA8I111wTZpxxxm7vqxoUARGon8AYY4xR/0baQgREoMcT4Hdct0w9/jBrB0VABERABERABLpIYIzErIt19NrNudn85ptvwj//+c/wzDPPOIc555wzTDDBBGGNNdZwzyUl+e61p4d2XAREQAREQAREQAREQAREQAREQAR6FQGJTL3qcGtnRUAEREAEREAEREAEREAEREAEREAERKA5BJT4uzlcVasIiIAIiIAIiIAIiIAIiIAIiIAIiIAI9CoCEpl61eHWzoqACIiACIiACIiACIiACIiACIiACIhAcwhIZGoOV9UqAiIgAiIgAiIgAiIgAiIgAiIgAiIgAr2KgESmXnW4tbMiIAIiIAIiIAIiIAIiIAIiIAIiIAIi0BwCEpmaw1W1ioAIiIAIiIAIiIAIiECPJsAg1RdddFHYfvvtw/LLLx+GDBkSzj777PDjjz+2zH7ffvvtYffdd+9yf5588smw0047ha+++qrLdVWq4O233/Y2XnvttUpFWmL5OeecE04++eQu94Vz5ZRTTilcz7vvvut8XnnllcLbdLYgI4hzvEeMGNHZKkq2K9f3Dz74IC3zzjvveHuvvvpqukwTItCuBCQyteuRU79FQAREQAREQAREQAREYDQRGDVqVFhnnXXCNttsEx566KEw/fTTh0cffdQflJdbbrnw8ccfj6aelTb773//O5x33nmlCzsxh/CDKPLdd991Yutim8CMNt5///1iG4ymUv/85z/DjTfe2OXWb7vttnDTTTcVrufTTz91Pgg2zTaOM8eiUYJfvu/33HNPmGOOOdLd+OSTT7y99957L12mCRFoVwJjtWvH1W8REAEREAEREAEREAEREIHRQwCh4YYbbgjXXHNNWG+99dJO8PCMV9OZZ54ZDj300HS5JkQgT+Af//hHwBuuqM0999zh66+/DuOPP37RTVqmXL7vzz//fPjiiy9apn/qiAg0koBEpkbSVF0iIAIiIAIiIAIiIAIi0AsIEEY09thjh5VWWqlkb/Fiwrvp22+/LVn+0ksvhbvvvts9Q6aeeuqw1FJLhcUWWywtc/HFF/uyzz//PNxyyy1hkkkmCWuvvXaYYYYZPGSJZbPMMot7T00wwQS+3bPPPhv4W2WVVcIVV1wR8HChPwMHDgx9+vRJ685P/PTTT+Hqq68OTz31VJhiiinCaqutFuacc86SYggACGmUWWKJJUrWVZrBW+Wyyy4Lr7/+eujfv39YccUVA+IC9tFHH4Xrr7/e+z/llFOmVdCPaaaZJiy55JLpMoSXK6+8MowcOTLMPvvsYcMNNwwTTTRRup4J+nXrrbeGscYay0W+N998M/z888/eJh5R1113XVh//fXD+eefHyaddNKw0UYbhd/85jeBPuKF9OKLL4bxxhsvzD///M55jDHG8PrvuOOOMO6444a+fft6HWOOOabzmWuuuUraZwavm6uuuip8+OGHfixXX331EOvpULjMggceeCBwLOAfLe4X/Oebb76wwQYb+HnGes4NeK2xxhreP449Ys2aa64Zrr322vDMM8/4ObLpppuGeI7EehE/ae+XX34JSy+9dOA8zRohnvfee2/417/+FWaddVbnmF2fn0Ygm3HGGcPiiy+errrkkkv8uGfr5nwYMGBAmHnmmdO+w+vBBx90ge2vf/1rSR1UxnGlr/369fN95TsgE4G2ImAXMZkIiIAIiIAIiIAIiIAIiIAIFCZw11134YKSDB48OLGH/arbmdCRmKCRLLDAAomJHYmJTImJEYnl9km3++1vf5uYqJRMPvnkiYkziXmrJBaCl5x22mmJCQa+jHUsM6HAt7N8PokJNsk888yTrLzyyomJHN6nrbbaKq332GOP9e3jAhN7koUXXjgxESoxcSNZdtllk3HGGSe54IILYpGEMvZgn5jYkpiXlk+bGON1Wx6dtFx2wgSXZLrppktmmmmm5A9/+ENCeRPhEhOLvJiFEvr2jz32WHazxESoZIcddvBllvfJy8w777yJiUvOg35aWFViOXvS7dgn2LOtCTGJiUg+baKSlzEB0NfDA44mFCUW9pU8/vjjiQlaiYkjycYbb5yYsOblNtlkk7TuddddN1lwwQWTqaaaKtl88829ftqy3FslZeBjIkhiAlxaz5577pmWKTLB8eZYRzvmmGO8r9TLeTLZZJMlv/vd75LI3EQk768JQb4Jxx/mJtAlJop5XRwz+JloFKtNOB/YB447+8e5aPnD0vWcTyZUeRn6Q7l4vE28TMtlJzgv2Pdob7zxhm/P8YhGvznPTSBNsn03Yc6PF33iHDSPwMTENd9+kUUW8X3hfObYcRwsB1WsUp8i0BYEUFBlIiACIiACIiACIiACIiACIlAXAUv+7GIND8s87CNK/P3vf08saXJaj+W2ScxbKNl2223TZTzUIwzxMB8NkYlyb731li+Koox58CSWo8iXWX4lfxBnHYbIQNt//OMffZ5/5i3lD/bmaeLL8iKTJSl30QlRINrhhx+eTDjhhIkl3vZFlmvKhZMo7Hz55ZepKBAFj7ht/EQMow5LDB4XuXBhXjU+H/eniMiEyPP999/7dpYI2oUG+o1RD6LR0UcfnYpttA2HvMhkHjuJeZQl5r3k2yI6IVhlBRjYIYSwjxgiDHVFcYxjNXToUBdmEKpiGbYZPny4z/OPY48oYt5U6bJaE1mRCQEMQe2www5LfvjhB9+U4wFT6sayQg3z8fjvv//+iXlEscj7Tf8t35PPcx4wj7ATDZGKZeZZ5os4jxEaH3nkEZ9nn7fccksvU0lkQnQzL7LEPK58G/NISiaeeGJnaZ5KvuzCCy/0ZRzLfN9PP/10L+sF7V8UmcwzKuE7g8GbfnKsZSLQTgSU+Nu+uTIREAEREAEREAEREAEREIH6CDBqm4k1PkKYeSl5jiZClUw8CoTHYYRkPffccyWjkZHYmlAjwp+yRtgbYWYYdbCteXQEwusw81DxMDgSjWfNPIfSWUKVCD1jVLlyZiJYoI8kKreHNv9jdDwSPd98880+f+eddwbzLvJwJeow8SDsuuuu5apLlxHyxohkJlh5KBt1k7Pq0ksvTcsUnSBEjJA1jBDBrbfe2sPBmCecDdtuu+3S0DTCE2GVN/Pg8fxFJuD5KkK87rvvPg+xYwEj5ZmnjO+ziUzp5oSarbXWWj5vYlLYb7/9PKzt/vvvT8twnGIZFhKyBsP//ve/aZl6JgjhI6zv4IMPTsPjpp122jBo0CAPyTPhp2J1HJsYHhlD1WLydPgTrkbesHi8CZcjPJIQSwymhOwtuuiiPs8+H3DAAT5d6R+hgdRnHn1ehDo4Tia0ecgdC0lqvuqqq6bHslJd2eUc13gsCbEjTDF+l7LlNC0CrUxAIlMrHx31TQREQAREQAREQAREQARamIB5MIXddtvNBRVyAZEzB/GIfErRzDPFxRfy11gIVLCQMh+RjhxCWbMQrHSWXEDmXRIQr6KxjDxQWcGB3E3Z3E6UpX7zHImbpZ/kEEIIIg8OdcU/ct9QJ8PHI5qRXJoH/KzNNtts2dkO04gYe++9dzCvomAhXgGBxLyEOjVSHMJa1tgf2CIKkbOI+sklFQ1RKJvTKS5nu6yRv4m8QeSt4riRo+mss87yItljYSGEJcIIgheMskzzdXNcMYSmzhhCCoIafcwaIo15Anm+rezyOI0gRP6oaAiCGDmWMI7pyy+/7CJUPN58vvDCC76OMuxX/njTF8pVMvhzPiNmcu4gNiGSsozcY5zzrGMExnqsHNfOMq2nXZUVgUYSKP0WN7Jm1SUCIiACIiACIiACIiACItAjCZB8mYTYWY8PBAKSTZNcGu8gC31zLyAevPHwsJA53wYvpV122SX1AomAovdOnC/yiQdLXpjAKwfPorzRB8xyB7knUH49QgllEC4QmrIWRYvssuw0gsTxxx/vnjh4tVi4lnsxkbD8P//5T1qURNdZy3oQxeWIa1kjSTdsLHTME4AjuuStXP/yPEn+jScTnj977bWXJ0jH28bC0Uqqy2/HyjzT6DlUsmEXZkhsnmceq4MHHld5zzfWc6z4q2QcTwQ4y//VoUj0GLJcXx3a5jhlxcwOG9sCPLkYRZEE7fAhmf3TTz/tydbhzHHKJjUvV0d+WaO55uvXvAh0B4HK8mx3tK42REAEREAEREAEREAEREAE2o4AHkR4weBhkzdGecOjBKEHrw5CqAjV2nfffV1kQsTAIyfrPZOvo+j8Z599VhJOhAeJ5T0KCy20UIcq8NzBw+hfNoIYYUjxD/Fm2LBh3k9CthhdLBsaRkWWC6lDfdkF7B/CGW0wGhyjup144omBUd/wpqEN7JNPPkk3YwQ1yzuUzscJy08UJ/0TwYpR6hCy8NrCM+d1G8EuGseA0f6qGWUIB7ScR76veAjRV0QRLHssnnjiidQTiHWM4obAU44p6xth7B+c8jwYFY7jlBfeirZpub/82CFSxeNN+Jzlc3JvOupB9IyjvcV6OYdqGSITx+Hcc8/1UDuEMkYUtETd/t0gdI/zqZxFYYyQO5kI9DQCEpl62hHV/oiACIiACIiACIiACIhAkwkQIof3hiWp9nxLDP/O0Ot4yBxxxBFhxx13dGHARsvyUCWGuidUjZC1PfbYIyCkWFLqhvSSnEyEJiHg7LTTTu6BQm6bckbOHxvFzftpiZVdZNliiy1cpCHnE2aJpF0kQpCx5N8uSCAYVTPC6c4444xw3HHHBYaof/755z2PEkIbohXhWIg6Rx11lOeosiTmAc+icuIJOZDwhMGz59RTTw14Mh1yyCHePHmWCCtEwCA077zzznNPHdhWM7x16COCFfmKCL2zxNRprqzs9oiCMCWkDMGEnE/LWghduZC8Sm3CmBxFfBYxjhd9JF8WbbLvlhw7WAJyF++K1FGuDMcSIccSebuQxPHk/Lv44otdXGIbxEHEQNpGwLOE4O6JV66+7DLEKkuk7hwRlzDCO9mPa6+9tmqoHIIU/br88ss7nccq2xdNi0ArEZDI1EpHQ30RAREQAREQAREQAREQgTYggEcN3j2IJ4ceeqiLEIQG8XCN0PKXv/zF94IE20ceeaR7MpGAGoEEceqCCy5wLyi8V7piePessMIKgUTMU045ZUC8IeF2TCCer5uE3ghGeBoh/gwcODDglUVC8Bg+hahiI6p5iBW5i/ByQqyoZohpNpKd5zxCWMIzB0GD0Dn6SNgWog4iBt41tPv73/++bDjVQQcd5Im08QZjGo+xmNsHUQqvG7xv4Erf6C/tk5upkuE5c8IJJ7gQhzcXoYE2QpqHLBKiZSOrpZsiHCI64flDX8k/xHElH1ZRI1SS+vksYuwrCddHjRrlibqZP+mkk1y4I8yys0b/8eBCqEMk43wkvI1jgUCEzTfffJ4EnHNn9tlnD4MHD/Z8WvkwzHJ9wJuJ0DrOQYxjjQAI72xi9Py25N2i/U022cRFzPx6zYtAOxMYwxRU+ei18xFU30VABERABERABERABERgNBLgcQIxgZCwbELqfJcILULgKOe9ky9bZB4vH/IrEeqFKIL3SxyJrtb2sc+Uj+JSfptYBmEihjfly5Sbx1OIOvFcyht1vmHJxWtxYJ/wrkGUy+bpIZwMLyRGR8saiaoRohCSalkMZ0TIyRsJzAnjI5H1Rx995PtRrlx+u3LzeCfhWVYpzI4R6cglhXdV1gjN43jCqJGGhxks8TSqZPBFrCyXl6rSNl1ZTrgnImf2GHelPm0rAq1AQIm/W+EoqA8iIAIiIAIiIAIiIAIi0KYEEGCyI8NV2o38yFmVynVmOQJXzHtUZPsifS5Splxb5ZKOx3LUWYQDokN+xDPqwNsGcQbvHDzHEK1IPk1ycUY3K2KMFFfEEFs6a4QLEvY2//zzl62CpNoIOnhL5Q1xrpxAly9X7zx5mWoZnmvdaXj3yUSgpxGQyNTTjqj2RwREQAREQAREQAREQAREoEcSIGE3+ZLWXXfd0LdvX/dqwvPn5JNPDiuttFLL7DOeVQhh5ULOhg8fHoYMGeKhcYT8yURABHoWAYXL9azjqb0RAREQAREQAREQAREQgV5BgLA0QsoWXXTRXrG/2Z0kNJBR8gjLI7cTIVeNMHJGEaqXD8drRN2xDryuyOeFJ1al0ddiWX2KgAi0HwGJTO13zNRjERABERABERABERABERABERABERABEWg5AhpdruUOiTokAiIgAiIgAiIgAiIgAiIgAiIgAiIgAu1HQCJT+x0z9VgEREAEREAEREAEREAEREAEREAEREAEWo6ARKaWOyTqkAiIgAiIgAiIgAiIgAi0PoHrr78+rL/++mHFFVcMzz33XEM7/MEHHzS0vtFd2RtvvBF22mknzyFFX26//faw++67V+3WqFGjAkm9o+29997hlltuibNd+qzFd5999vHE3V1qRBt3isBnn30W9tprr7DccsuFAw88sFN1aCMRGJ0EJDKNTvpqWwREQAREQAREQAREQATakADD02+44Ybhv//9b/jd734XujLcfX7311prrXDOOefkF7f1/IcffhjOPvvswCf273//O5x33nkV9wkRaN555w1PP/10WubCCy8MI0aMSOc7O3H88ceHzTbbrOrmF110UXjiiSeqltHK5hDYc889/fyfbbbZwhxzzNGcRlSrCDSRwFhNrFtVi4AIiIAIiIAIiIAIiIAI9EACzz//fPjpp5/CxRdf3PCRyB5++OGw8MIL90BqxXfp448/Dgh5zTBGdmMEuWr21ltvhbHHHrtaEa1rEoGnnnoqrLHGGuHcc89tUguqVgSaS0AiU3P5qnYREAEREAEREAEREAER6FEE7r///jSU6rbbbvMQsFVWWcX38YUXXgi33npr+Oijj8L8888fNthggzDWWKWPHGz/2GOPhffeey/MOOOM/kDNJ3bBBReE77//3r1oLrnkkrDFFlsE2sBiG0y//vrr4c4773SPnPHHHz/885//DBNNNBGrPBSNsgMHDvT5Wn369NNPw2WXXeZ19u/f38P/5p57bt+Wf9dee61Pr7feeumychPV9qtc+UrLYHfVVVf56htvvDH88MMP3qdYHi+om2++OYw55phhpZVW6iDIvfTSS+Huu+8Or732Wph66qnDUkstFRZbbDHfHJb/+c9/wjfffBP++te/+vH57W9/G6tOP6+++uowzzzzhAUXXNCX1WKUbvjrxB133BHGG2+8MMssswTCKvF4+/3vfx/WXXfdkqIIlbSFsDLFFFOE1VZbrYNo+d1334X77rsvPPjgg4Hjsuqqq4ZJJpmkpJ577rknPPDAA+GXX34JSy+9tIeaZQu8/PLLfhw/+eSTMGDAgLDOOuuUeN/VWk8/hw8fHhBXv/3224CX0UYbbZSec88880yAO+IoXmDzzTdfmHjiiQPtDR48ONuVgMjHMfjDH/5Qspx6//73v4f3338/cEw4PrGfhE5efvnl3j7HlPN7rrnmSrevdv6nhTQhAt1FIJGJgAiIgAiIgAiIgAiIgAiIQEECp512WrLAAgsk9rySmMiRHHbYYb6lhbgl44wzTjL99NMn9mCd2MNwYsJCYjlm0pq33XbbxMSHZNlll03WXHPNZMIJJ0wmmGCC5Mknn/Qy9lCdmAdNYg/xXgcLTVRITHxI62DCRBhv34QqX05d1Eld1GnilC+v1Se2n2666ZKZZpopsYf+xIQxb//KK6/07fln4YD+ly4oM1Frv0xY8P6auOZbH3vssd7XMlUlL774YmLCkJc3cSgZNmyYFzPhwfvB56BBg5Kpppoq6dOnT3LNNdek1Zx//vnOl+NjIogfgzHGGCOBA3booYcm00wzTTL55JM7U8sVlW6bnWD94Ycf7ouKMMpuy7SJSYmJPUnfvn0TE1ySRRdd1PfH8lKlRU1MS0yU8X3g+HL8OH9MaEzLmPCSmFiYsA8rr7xyYuFjXsaEtrTMVltt5XVTF+1yfg0ZMiRdb4KbH9OFFlrIjzHHG3YmCnmZWus5f6l7sskmS9Zee+1kySWX9D6bgJaYAOZ1nHDCCYkJpYkJP37+MW3hkd7vPGP6sfXWW6f9ixMmSPkx4fydYYYZfJo+wn/mmWf2ukxcch58R04//fS4qX+Xyp3/aQFNiEA3Egjd2FbZpr788svEEtglBx10kP9IbbrppgkXXS463WmWXC2xNwKdavKuu+5KLHY2saSH/mN49NFH+8UgX5mp0vlFPXZ+5MiRyY477phwfNvV7G1B+uPayH34+uuvnY3FuTeyWtUlAiIgAiIgAiIgAt1GgPskRCYejDHzzPCH+1133TWxUCxfZnmF/GHeElz7/OOPP+7b/O1vf/N5/vEAjVBiiabTZebRkgwdOjSdLyoy0R/z/PH2v/jii0J9QjDjof6rr75K20Ow4pkkmnnZJPxVsiL7VY/IRDvPPvuss7r33nvTZhGXLPdVYh5KvuzHH3908QHxBUPwgB2CVzTz7EnMI8lFkrjMvMsSSyodZ8t+ZkWmIozylSD2cDyyz1d//OMfXeyJrLfffnsX2rIiDMIWx+Ptt9/2KhGLEMXM0yhtYplllkmWX355n//HP/7h7SA6RvvXv/7ly8yDyhdZcvqEbaJZMvVk1llnTdgvrNb64447zoWtV199NVaRIEKyfzfddJMvQ2Rift999/Xzzzy/Es5B87JLRUIKWoJ8L5c9rmmlv04gpPE9ioawhWgVRTGW77HHHs7SvPS8GOds/vyP2+tTBLqbwGhN/E3iO1wKSRqIyyyJzXCH/Mtf/uKJ7uzLZ9+V7rHOJtI76qij3H3VLjTucmk/lMHeEARTqEuS5eHC2ZsSt+GeS3JDjme7Gi7PV1xxRcO7DxPYwEg2+gn86U9/8tFeGPEl/7f//vs3vIP5kWIa3kADKqw14kwDmmiZKviO24NMy/Qn3xHCGTgvG5HoNV+3eQ143Xazn1/VlPnedF41BaAqFYEWJ0DIk4kegRHQzOuEF9l+b8zoc9wnY4RemaAQTMDxecowepp5u5SMouYrO/HPPKg89I4wMsKpivTJBAwPHTNxw0O26NMNN9wQLr300rQHhD7xV8mavV/ZdgkVM68WX0QYImFThGlhhKcxyt/JJ5/s8/wj9MoEii7xLcIobTAzQUgZ4W/RGC2Nc4QQMozQMM4Fjhvc+TPhyZ8fCAfE7GV+2GabbTw8zRfYPxOWwplnnumzHCfC3whljHUQLjfnnHOm9/H0n988E5UCuaYmnXRSz3e1yy67eB211pvg4/fthP5h7EMM18uO/sc6e8nuYYyEu1GGfrGf0Uxg9eNHCGMRoy1Y7Lbbbr6fcRvzhPJ+XHfddXGRcySXUzz/0xWaEIFuJjDaRCZTdoOp7h6rSgwrw3ieeuqpwdw9PdaUiyEXlFpJ6bqZV0lzxCYjKHGBMrXdL3imnHt8OUKCKdlpeeJ32WeZCIhAaxEgbp5cC+QCyP/xvW2klRspppH1N6KuIiPONKKdVqmjWWJyo/avmaJ0d74M6IkjRTXqGKseEegpBMzLw+/buYfnITf+nXXWWX5vTL4ZlvGSmQdkRqQzjxXPm0SS60bc81vIWwnOIn1CBEAYQ4CgT9NOO20wjxsXZ0oqqzLT7P3KNp3fRwvhKnmpS/4mBLPFF188sI7yDz30UJf4dpZRv379sl335z4WIJzwYj7mhYrnCp9sQ14ljh0vQci9FcWdWBllZp99dp+lHPmUzBsuPeeohzxcrMOOOOII54FQY2FonmvKQhC9H0XWjzvuuM6QfFIIWhaS6ecw22bPW/qAYJY1znXuJy3Kw/cLUYw8YwixRYzfavJB0W7WED05V7MvrfPnRra8pkWgOwmUZuHrxpbN7TBYvLGr7fkvhMXIhpNOOinw5oNhOy2mOJRLpmaug35BQb2vlNyOXWLUCy60KPkkoePHD1Ud1TpvtRLpZcvzQMqFZfXVV88uDuZe6m/GEc64iHJhIVEd6joJ3OhLTCZIn0jsh6pubpuudmeHgOUB2Nw7/cJGwjwSzPH2olaSPMqS2G+FFVZI33aUdNJmiibkq5VojnoR0Eg4B5Mlllgi35TP1+ozhfD44o8fF5INcg7UGtmiyPG32Gn/oYIJP7689SH5YNY4Brwp4EbHXE6zq6pO099qiQY5B/DKQ4DkGFtYZYf6KEMdnMcknCRBoLnfesLMeK6wUa22aiUt7NCwFjgBjne1YYQbhamZI8U0qo9FRpxpVFuqp/cQ0EhRvedYa097LwGSbyMacc9b7t4tPqjjzYIXB16a3DNyn5N/gC5HkfvIrFlKhuysT9NG1or0CUGCFywHH3yw3xtzr4gQYOk8PDkz62sZIk5n96tW3fn1CBmVzHIH+XMG+20hc86XqBFeiOMR1FnrLKNqfaWPmKUcCdttt12HriGQxfMIgTJrPH8h0tAv6uE+33JOZYv4NJ5dGHWRjP7NN9/05yPu93EU4BmC5PK11nN+4DyAYIRgxXlL25bbyeuP/9jf/D5bWJ8LW3hf8bz0zjvvhC233DJuUvMzJrNHkMsbz5Y8N0TLn/9xuT5FoNsJ2Mk5Wow4YBLaFbVyydSIMa6V3I76iV9e1hLJkXiOGGWSENoX1mOdY/uUIakfn5US6cWy8ZPcOnZhS8xFNrGH/8SEgriq5JMYYfsB9ThZktqZC66vtxExEht1wGORiY0mNpg4axMb0u1ZbyKWx6qTxI5EeEWS5JHnyk4mjxdOK8tNFEnIVyTRHP0hOZ1d2BJ70+HTJE2kfWLxsSJ9PvLII70OYqzpm7mYevx4Ja7UW/T40y/4EY9Nkj77YUiI4Y5mN0Qea00SQI4F8ezmZut/sUy5z61qJBrkHI0x0uTsImlgZGNhOl6l/VD6MYYX3wvOT5IG/uY3v0ljxSlYq61aSQvL9V/Lkg65C6oxgbHdXHiuLqbzxrlqrvlexkLt/PyMeQfszW1ymCVG5TiTd8JEXt+cc9jE2ZKquAbE84MVFs7reSXspjch55u9tUrL1+pTWvDXCRNZExtBx3MGkEiU61A0uwHz65m9HUxs2NyEfALRuKYccsghniMjuw3r7WWAJ2C1t42+jyQgtYcB35TvPslGLSzRE2AynzVyVNCWeexkF5dMm9ie2NtITwrL/pOng+3yxvXKPGKdL8k2YZ43u8FMzMXejxFJZnfeeecO3/Nafc7XyXy1Y896e1Hix5QEphzH/fbbz/fbbhpZnRrHh3ODPIUcd3th4OdM9nxIC2cmaJ/jynlnN8OJvdzIrP3/SXJQUM+BBx7oOTJi0tx4naYUZVh+wAEH+Cfz+XOhSFvZxjnH+c3lWmgvfdJVteqxFzWJjSaU2A15csYZZ/i+mddhun2c4DvAuQkzGMAwazZKVcI5RJ4M7iXIh5E1e6D0312+dyTEtYdFX02eGZKqmodDYqEnJfcM2e01LQK9lUA+J5O9SPXrlY3AVYLE0mD495OFJnYkFqqU5mximT14+31uTNTNMu6H+c2Mxr0ZyZKzRj5VflO59mNcY7iHz1qRPlkok/8WlNvORIjs4orTRfar3pxMMXcPzxjReE7hPiRrJo75vSPLuAeBST7v5yKLLOLJ2ON2lqrE74njfLnPbE6mzjDiXj7mTYr1c72lf6+88oovMk8cTyAf1/Np4ZTJJptsksQE6dy3b7bZZtkinr+X+2V+Q8mdRRLs7P0Fvy/ke4q5v/h94Hcka2zHsyFWaz05rSwMLbu5/46yLzFJOb8vJC0vZ/y2kLibPEs8k9aybE4mniX4zpgoW7IZyeFJhh7zTpU7/0s20IwIdCMBvGu63Xj4IKGbuSwWbrtcMjUeSookt+OCzJeem3WMCxJiDw9S0ShTLZFeLJf/JGmbqd9+wUQU4Qt+yimnpD94sTw3qlwIovHgycgGXLBiwkRuhBlBArGJaQyRiYs8F1z6zfIiSfIYBYEbZfOciE12+OTiz8XR1Px0XT4hX5FEc4wCgiDDgwBGsu8oqsWHlyJ9Zj+zNxQWxpLw41PuYZJ26jn+nCc8XGLmmeWCIqwx6jGvNk/aHh/4eEBDNGO/KlmRRIM8mHDuPfLII14NPxRbbrmlc48PjfHcRujC6A8/rhybmJCwSFu1khZ65frXgUD+GtKhwK8Laol8tUYeqTRSDEI1N+BZQ/jIXp/Kic2Ur9WnbJ1Mc/4xChDXLL7/iJqWzyGJCVm5CeK7wHeRayTXHURQrlOcjwjqTLMNN2fUh3EOm4doyYgqrOO85yaQ/jPCDeIp9Wdvfnnwp+5ygpBXbv+4PtcSiouI9kXE5CJ9jv2Kn7WOPeX4XeCaz4g03NQiOnONmXfeedMXFDDjNwQelUTp2Gb2kxtrBGx7e+rHjRvY/Og8lJmhwMsARoSiXxx/hHnzvPX+IJJhRdrK9o3pciNFFamHc7TWyES1Xk4UeVECc5hlR6RiO46XeVpXHG0qv5+aF4HeRiAvMnH/wm8B907c43B/i8jD/S+iOYbIzjUOwRghgKTFfP9Yxn1MNL57/F5xL4jxgoEyvLQgITQP9vy2sKyayFSkT4gZ1MNvMfetiDskGuf3it9AjPtT/ipZkf2qV2TivpV+8WKKJOBYLZGJbfgt4EUGL8Pfffdd/42nHu53oplHjv8WMXhRpUF6siJTEUax7vhZRGSyUErfRwZQ4mUSL9247nMPxLHDOIe470AI4tgjVFm4XGJ5cX09v0+s53eLlwWU4T6K37KYFJvE3bzsQABF1KQcv3eIbVit9eZp5QnsOTd4kcHzH32AaxzhrZrIRFJ8vgecs7z0qGVZkSn2DyHNvO79uNIPhEPEL14GYRKZHIP+tQiB0SIy8XacLyUXi6yRnZ8HEv7wKOGPiyQWH8T5kmaNH4PoKcByLqY8BCHUROOCbGFjcdY/uajSBx6iMMpk36CwDMGFH8taRvv8uGy++eZ+U0q9KM7RY4nt8yITb14pxyhsWeNBhOXxTSoXo3y/EOgYaYEHkvjHfuOdE4cnzdZZaZqLP8PDZi2+2YYLF1Eu2ieeeGK2iP8A0Ecu7rTPRRsxJWt4EVAmikxF+owwxY8KfeChrYgVPf7ZYUyplx+feGzjiCDZoVApw9C71UQmfszMvdtvQOJx4JNtGAIXwwuNB6ysITbAJopMPEwytGvW4vkRRaYibbFPCJ14ckRBLVunpssT4KaL6wXf3/xfvMEpIvIVGXmk3EgxRUWmvNhcpE/5PeZGjHMv+1aUG3fEn7iv+RFnLHTZb1jxZIrGWzPqwUMFi9dnBCNuyLnG8/BAvQxdzY0uxjLOd8SqeOPOucr1jnWVjOtzNaG4iGjP/tUSk4v2Od/PIsc+XtvxNIpeXnFkGm6YsSKidL5t5ouI+EVeBnCDynUxvjBgRFSOIcc6ikxF2irXR45fdqSoIvXwG0Xb1V6E1Ho5UeRFSRT2siNSce3ldyt7f0E5xFWZCIjA/xPIi0wsxUMFr3EeqPn+cl+Hp000XvTxPeI+mYdm7lsYVRoPDwv7SV9eIPggllMHIgjCAN/nWC/Xpui1U01kKtInytAHHthj/dyXxese64l24K+SFdmvekUm7im5dtOn6MVVS2Sif9yfwxK+/CEoIcrBMnoQMfoaHvMs49pXzrIiE+trMcrXUURkYh95zuA8oC+I/TzHZV9GUYbzgWsyZfjjxVO8t6BdS9nhnkKs43lo4MCBJRELvKS3XL/+8poyPN9YrsD0eaPWejzaED0jU7ySEKwspUv6u1BNZKKPiKn53xWWl7O8yMR9w5///GfnQ/95CWSpP9LnLOqQyFSOpJaNLgKjRWRiZxE3+HJkjaEpcfOPf3wREXowvrgo8/HmPG6HdxKu7FxMuPCiWvOWnjcg0Vie9xRArOJLGi+slDks94Yi634a6yryiWjBgxU/CvFmPS8yReU+e4GkbhjQr+jeiciU7RdiEusr/RGSUtS4+HMjkLWsGyvqP+3EoTmz5fAwIvQwcswKapSLIh4iUNE+I/Ig2tAmF3/eZBD6UM2KHv8sQ+rjQY99wOIPLzcwWSP8oprIFPta7ljwdgHjLQnu3FnjQZYfwCgy4UGXL4PAh/gQRaYibfHGkO9U7A+iHT9I+dCRbF80/f/hcryNQvzI/xHahBUR+bhBid6SbAP3+H1ChMa6IjLlxeYiffJGM//wFuK6hPjJd5Sb4rzlRSa+h9yIZQ2BCNEghiVEkSn7EiC+9WTfuUGMf5z3nKNFQxBol+tzNaE4irLVRPsiYnJn+1zk2EeRKf4msF9488AivtUsIkqzXd74raz24gH2tV4GcF3ipjXvYYx3G32MD1u12sr3Lc7nRaYi9dR6EULd1V5OFHlRQh3cmHOtzlp84cL9Bb9NMJSJgAgUJ4BAa7kuK27A78/rr79e9btFmfy9GV4b2etoxQbKrKjVJzZBsCr3opPfd8SPWlZkv2rVkV9Pv+OLoPy6avP8Jpf7nY/bcF2L0RRxWZHPSoyKbFupDH3hBXe1/eTeA2+n6LlTri6eO6pFcbAN5x2/25Ws2nrODV7AdMZweEDw64rBgJQJ1frflfq1rQg0isBoS/xNwjQTeDwJs72psHvY4ENTMrpDNFPM46R/kkiNv2j2RS+c3M7Ep7iZf8ZkgfZmO12erTtdWGXChAtPXM7wqFkjwZ49WHkSQwvh8BEEsuuZtht+X2ThWZ4kMb++UhI3Ex68aLUkefm6qs1X2+dsH/N12AnoieZIlGcPrcHEspIi9sCSzhfts+Uq8gR8DDFqD+eBITnNIyiQ7I+RBvNWz/Gvtp/2psar5ljYm5S0mew+pAszE+xXrUSD1J1nY0KpJzaPVcHZfizirH+aCOXfjbiwSFsci2pJDWNd+uxIgOGAqyX+ZnSSOHJJfuv4PeEaYx45wYRRvy7YTYoP3Ux5uynIb1b3fH6AhCJ9yjdigmZg6FxGV2FQAPrOKJ/mMRpMGM0X93mSYpqHV8k6zk8TmIK9lU2X8x2zB/V0nv5hlkfB/9IVv06w3sTT/OKK84wkkzUT4NNRYegjlq+P42qhgD7ySvw+5zkyvDIDFmCd7XPRY8+1kmG6o7EPWOybCTnBRL642j8ZUQfelcxu9tPReSykoEMx9slu3P06FIe8joXY92ic31yHTGiNi/yT40ySU6xIW16wxr966il33Kk+MmNYaAbEsJCHYC8n/J6AZMIMntCVEXnsLXk62pSJqH7czKPAE8Vm7xtq7KpWi0CvJcDvS/x9LAeBe3/znCy3Kl1GmfiMEBeaB5QPPx/n6/ms1SfqqvT9ttDcsr9l+faL7Fd+m1rz1ThW2zb/e5cvy28S9471WiVG9daTLU9fLKQ7u6jDNL+F+VHm8oUYPKqW1Trvqq03769a1Zes5/6PP3sZ5n8Wkl+yvt4ZGOR/y+utQ+VFoDsIjDaRyd7KBwv38BtqRJr8j4h5v3R48M4D4QtrbxaCuVQGcyNNV/PAkH+oY2QbHqyiMWy1uTwGy4cRF9X9aZ4wfvPNg9aiiy5asj39x+JDDxdPDHGGaXv76vPmruo3yD5j/+6xkcowBCY8VW4AABUHSURBVJdyxsWNdtnO3EvTIhZyEiyZa9hjjz0q/kCmhQtO0A4/5vk+8kDHQwLDvLLeQo0CPBGEomUfPov02d6g++gSlmPGR9NjREFGceBCilBXTmSq5/jHfpX7RBTE2AceVqJl9yEuy34y+p2FuwR+0Owtva9CQLKwNR81hb5TN6OsxONOIfOWyFYTLDeU7yPnbBTD2OfsOVykLUZF4YGMYXd32GEH/+MBn7pkXSNQROQrOvJIuZ5w3mQtiuDZZXmhvEifstvHaQtT8AdyznfEXM5hRvlB3DXPvVgs/eTmFgG2nGXFcM7deP5Slv5hiG4Ms5u37Lb5deXms3Xn18cb8Gqivb0N982qicmd7XPRY8+1P/4W5PeB+SKidH672OdqLx4oQ7t5wTsKNdQZGeYF72yZIm3l+1duvp56qh136q72cgKBDCt3/sYXJV7A/uW/X9zIc1y5rjISK98VXh7VM9pUrFufIiAC7U+A4e7jvV777432oLsIWPoAf06gPZ4P8iNbd1c/1I4IdDeByq9Hm9wThvk0d3S/cbNwjIDXEmKGxXe7UII4w80lwz5WMt68U4Z6uIlE+EBksbAI937JbkcZbhYtXMSH8LTcEC5i5G8ss9vUmrYEhS4WLbvssl43w2EyVD3ij7n5u/AULybcwHNTy/4hjCGiWGiEvyllG27sLZ7Y68FroJpSzn7wQGghVv6m1kZ2cm+pESNGpKIZb8QZZhMWnTUeSmgLDw/LveKMLSG2ezUgkrHfmIWe+Xo8u8yFOVhYSIkARplafWZ/LXTEhSa8QXggRHy05H3uLUQdeavn+Oe3zc4jzMCchzRLTOnDm+L9EAW/bNnsNPvNMd1yyy1dSGLfOf9s9CQXlyjLkLEIgDzY4ylguWyC5SLJVhMsN5mX4UGffYYj4lDWirQFQzxSLHQxIFIgbiEk4G0VjZsk2pPVRwCRD9ERQdHyQPgf1yjOdc5XjOPOd5rvCl4UiCjRQyYKhlFgMLfwtAMWNhTMXT2dZ8ISOpbMl5sp0qf8dgjyiMHmeu+eTJZLyK+H5k7v5wrl6WO2f3zXbYSvkmUID5xfCM2VLArpiKqRGZ94ePHwnhfWKtVTZHlsi9+QrMXvMEJEVkzOlsmKybGeevtc5Nhn26w0nRWlY5m8KB2Xx8+siJ/lzHnF953fm+zLgLgdn9l953zlxQK/Q1nLzhdpK7ttdjp7XnWlnmydFjLhv3P8hvKbaiPiBV4o4VGHuJ59UZLdLvuiJLs8O81LMK7f9BUvKb7X/LZzPY8eb9nymhYBEejZBCQw9ezj26y9w4Oce3t+pyxnarOaUb0i0HoE7CF5tBpDNW+88cZpIjYj5LkRGKaRfBXRKiVTK5LcjnwejGxEHijqt5tGzztB/G80ylQbEjSWy3+SjJAEhiQrp27+SJRHvqKYz4VtiN8liRvryfGAEQfNKGImlKXbkYPKHr58Pf/IycQ+Zo1+10qSR5Je2iKxbCUrkpCPvtRKNEf9JA+ODMgxFEcBiYm/i/SZhNgkdYzJ/8jZwTDa9oBeaRcKJTcsd2yzOZmo3EQ+P44kHIQbiYlJpF0tJxPb1Uo0SBmS1lIP9Zqo6aNMkXMq5mSiDCNaMeoU+24PxD7qCuVjYmXK1GqLfaiW1JA6GFGP81D2PwLkiuH7Ws3IR8MxIw9SpZFLiow8QpJrjmt2pBgLV/OcQyaIeI4zrg+cJ/nR5fLXgSJ9yu8TeR34nnLN4vtGvjQGVzABIB25Jj/ijAkRvp7hmbkmcy0zj0M/V2MOpErXZ67tXG+5XpEPijxQDOhgwmzaNa5R1Me1tJIV+Q6Tz4jRwExo8pwO5MPiWkQy92hMm6jsedgshMx/Bzge2e95kT7H+uJnkWNPTibzjomb+CffWdpn5ByMUXXshYQPHGBCSGIhw34eUCZ7vfDCmX8xx1+10Xm4lpjHcEKOJUbeIcm4iU/efrxOM2oNfYQBufg4F9iG9jnfsCJtZbqWTlrYRslIUUXqKfIbxXEn8SojB5Kng0TE7EMcqpqk7CRqrXdEHhP3fL+rjTaV7pwmREAEREAEREAEREAEUgJ4YrSM8QDGqESdsWrJ7bIPKLRhb+E700TNbXgA48GlmrF/WRGJsjxokMQtK3pVqyOuozztVUuSF8t29ROhp1aiudifavsRy1TrM8eH41lNXMrvT7Xjny9bbZ5+8QBWrxVJNEjSSI513hiGNN9mFCPyCdXZtkhb5i1Stq1825r//8TftUQmONUS+RAFao08wvmfHynGvAM9eTEP8jwck4QYQamWyFSkT+WOL4MdkMybB2/aRHQiyXE0RBqEIdbFgREYTc68QnwZfUSE5aE+WiWRCeEIASa2FQWmbKJRRqSjLUSoSpa9hscyeaG4iGjP9w+BrZqYXKTPsQ/xs8ixLyIyUV8RUTq2Gz85r2q9eKBsrZcBlEGMYlQlBG8S4TNaJceH6z9WtC0vnPmXHymqSD1FRKZaLyeKvCipNCJPvSMpZXZXkyIgAiIgAiIgAiLQawmMwZ7bDWSPNpLaEQplo4X16P3UzrUnAfNq8VAMG3nMwzvsQTiY6BHMk8BDizqTlLE9SbRHrwm5JUw3JozP95oQHsLPTLzJr0rnyY1DkmJykUUjjxshToQ21Wu1+lSuPvpAWCohRXnjZ4HE+vlzjz6ST8dEn/wmVecJryNsa0ZL8moiVdWyXV0Je0JXSXgawxPzdfIdI0Sx3L7Hsp3pc5FjH+uv9UmoMG729YR0c9xsNCc/97LnVratWIaQsjwfQvNM3Cw5By+55JJgnmaBfcsOjBDr4Tyv1Fa2XaZttDcPDe9qPfl6mTcxyc+xGSxxbLlzzEStYC9l/JjXw5S6zXvP95HQOZkIiIAIiIAIiIAIiEB1AhKZqvPRWhFoOgEeYOxNerDQI08Uz4MQifBJMMsojDIREAER6A4C5IRC6CTJNWKiDQPt1yDELgsT7Y4uqA0REAEREAEREAEREIE2J9DcV8otAuf2228vOzpZi3RP3ejlBBgKlgTteF+QkJ7kgCRBl8DUy08M7b4IdDMBErJbqK17qjHSIF5KeNxdc8013dwTNScCItAuBBjQhIFwLLy20KAVrbhfDKZjuWB9UIxs/yxfng/Ggmcp6y1sOLu64dN4C9POK6+80vC6G1Gh5Qb0+9RG1KU6REAEejaBXuHJ1LMPofZOBERABERABBpH4IUXXvCRERk9kPC5fFhd41pSTSIgAu1MADGEUS0Z3ZGRoi3HXjpcezvtF6N/MYqk5bws6T+jKDP68EILLeT7iEdnM18AkjZh3nnnDZYbMSyzzDIth5ARN+++++7ASNMyERABEahGYKxqK7VOBERABERABESgdxHAi4k/mQiIgAhUI4DYQD60iy++uMddM2w01vDQQw+FSy+91EOHq3HQOhEQAREQgVICEplKeWhOBERABERABERABERABESgCoH7778/2GiYXoI8bjYqblhllVV8/qmnngq33nqrDy5ho1WGDTbYINgoo76OXG/XXXedh9jZaJY+0ADJ+vGIWnzxxdMWGXSgf//+YbnllkuXXXbZZZ67Es8ijD4wYMF7773nA0usscYa/sm6cu1stNFGgQT+DHphI8a6x2YlzyQ8dgYMGBCmm266VGRiwIMrr7zSc2ji5Yn300QTTURzqVXrUyyEB9hdd93l9dJnvEYrmY3k6p5DlLvnnnt8/7qLEwNJcIxhSe5QmQiIgAgUJmAXTJkIiIAIiIAIiIAIiIAIiIAIFCJw2mmnJRYmxwjVyUorrZQcdthhvt0xxxyTmGiU9OvXLzFRJ7FRShMTURILRfP1I0aM8G1WX331xAYY8LIrr7xyYmJP2q4NgOJlTKBKl7G9he4mNiiKL7NReBMb2TKxkLbEBJDEBi1IJphggsTyK1Vsx3IqJZbzMrFRKBMbZTJZb731fHr++ef39mIfqWDHHXdM9t57b6+LOtlPC2VLTFxK1l577cRGmU3mmGOOxPJpehn+1eoTZSzHXcpnrbXWcgYbb7wxq5JnnnnG27FwOZ+3UYYTGwgm2W+//Xye/nYXJxO3vO2pppoqMZEwmWKKKRLzcPU/74z+iYAIiEAVAqHKOq0SAREQAREQAREQAREQAREQgQ4ELr/8chdFPvnkE19ng5i4+ILg9MMPP/gyS5rtApDlNvL5KDJZDqfk22+/TT799NPkoosuSmyQgcQ8jLyMJdxOJp54YheVPvzwQ1924YUX+rLvv/8+oR1En7/97W++jn/mzeRt77PPPr6sXDusWGeddVwoieLQl19+mcw999xeX1ZkmnHGGRMbOMjriiLTggsumNA+9uqrryYIMNtvv73PF+kTIhfC2P7775/yMe8kb/u+++4rEZnME8yFsCje0Uh3cfruu+8SG5QmGTx4cPLNN9/4/llopPcHoUkmAiIgArUI9IrR5Qq7damgCIiACIiACIiACIiACIhA3QRuvPFGD387+OCD0/C4aaedNgwaNChcddVV4Zdffknr3GqrrYJ5MvlolubVxEtvDyGjwB133BG23nprX08SbMy8esKqq64azAMpmNgTzNspbLrppr6ObT///PPQt29f//SFv/7LtkO5O++8M+ywww7BPK28hIlZYdddd81uEl566aVg4pYnM8+uIOyP9rFZZpnF+3jttdf6fJE+EfpGDqtDDz005UNy8SeeeCIQVhiNUbHXXXfdsNtuu4XDDz88Lg7dxYkE5O+//34wMSyYd5i3T54++iQTAREQgSIEJDIVoaQyIiACIiACIiACIiACIiACFQkgziC+mFdSSRnEIfMACu+++266fKaZZkqnLRTL8zEhriBEka+I/E7kHiI3knlFBdaZF5JvQw4nRCCEKPIZWahcGDhwoOcO+vnnn9N6mci2gzD19ddfh5lnnrmkzGyzzVYyT74mxJ8oKMWVFtYXJ/2TuslXRJLwIn0iVxVCGOJa1sgxNemkk6aLjjrqKBd3Hn744RJhrrs4Wdheun9pp2wizym7TtMiIAIikCUgkSlLQ9MiIAIiIAIiIAIiIAIiIAJ1EyAJNiJOObPcQsHCy9JVeQHH8hN5Mu6RI0cGC2ELSy21VFhxxRVdcLJQMhepVlttNd+eUd+WXHLJYKF4YaeddvLk3xZ250m60wZ+nci2Y/mhguV16tDHH3/8sWQzEpnnBSUKsA9Ze/HFF12IQuQq0ie2txDBbBU+jYiWtT//+c/h+uuvD3g+nXLKKdlVoTs4TT755N6mhcqVtJ3nVLJSMyIgAiKQISCRKQNDkyIgAiIgAiIgAiIgAiIgAvUTsNxGwXIVufiT3ZpR0Rg9Li/SZMsgnrz++uvh3HPPDYsuuqiP2obIxEhsZ511lo8yF719GGUObyC8mwh9m2eeedybiO3znkzZNth+1lln9VHpsssfffTRdBaPq39ZiF4cKS9dYROWdyk7GxCj2Ge8mIr0CQaWv8oZxYos/5GH7p144olxkQtoiGyWfDwcdNBB4eWXX07XdQenhRde2NtjpLysZTlll2taBERABPIEJDLliWheBERABERABERABERABESgLgLbbbddwAuGXEmIQ3g1nX766eHKK68Mu+yyS9W6BgwYEGy0tmAJvt2DicI2ep3XR96jGCrH8sUWWyxYkvAwfPhwz3GER5GN+OYCU977hvJZI8/Q+eefHyyhdrDk3+4plBV48JqypNeB/uTNRnkLrGe/Tj311EC7hxxyiBcr0qcNN9zQQ/UsoXawUfJcjLNE5b79kCFD8s2FY4891vefsMCYz6o7OJGvyhK1hz333NOPx5tvvhl23333gFiYtauvvtpDFvE8k4mACIhAloBEpiwNTYuACIiACIiACIiACIiACNRNgCTaJNYeNWqUizTMn3TSSeGMM84I2267bc368NIhMfYKK6zgZfEQWm655TzEjXXREGsQshBfSEyN5xN5n0jg/dhjj3kS8Vg2/7nNNtuEoUOHhnPOOcfD64YNG+YJrmM58jGVC5VjPV5Fa665ZmC/mMbDKopfRfpE6J6NGuceXSTx7t+/f7jmmmtc9KLOvLHs7LPP9lA8OEbrDk42wp+zRxy0kfbCAw88EHbeeefYBf+0EfyCjXgX8MaSiYAIiECWwBgMP5ddoGkREAEREAEREAEREAEREAER6CwBRnvDq4jR5Zpl5DIimfgMM8zgQlQ97fD489Zbb4Xpp5++ZFvC/Qirm3LKKctWRzgenj202adPnw5livYJPuSRqlRPh4q7sKBon8o1QfggIX7NPI7l2tUyERCB9iYgkam9j596LwIiIAIiIAIiIAIiIAIiIAIiIAIiIAItQUDhci1xGNQJERABERABERABERABERABERABERABEWhvAhKZ2vv4qfciIAIiIAIiIAIiIAIiIAIiIAIiIAIi0BIEJDK1xGFQJ0RABERABERABERABERABERABERABESgvQlIZGrv46fei4AIiIAIiIAIiIAIiIAIiIAIiIAIiEBLEJDI1BKHQZ0QAREQAREQAREQAREQAREQAREQAREQgfYmIJGpvY+fei8CIiACIiACIiACIiACIiACIiACIiACLUFAIlNLHAZ1QgREQAREQAREQAREQAREQAREQAREQATam4BEpvY+fuq9CIiACIiACIiACIiACIiACIiACIiACLQEAYlMLXEY1AkREAEREAEREAEREAEREAEREAEREAERaG8CEpna+/ip9yIgAiIgAiIgAiIgAiIgAiIgAiIgAiLQEgQkMrXEYVAnREAEREAEREAEREAEREAEREAEREAERKC9CUhkau/jp96LgAiIgAiIgAiIgAiIgAiIgAiIgAiIQEsQkMjUEodBnRABERABERABERABERABERABERABERCB9iYgkam9j596LwIiIAIiIAIiIAIiIAIiIAIiIAIiIAItQUAiU0scBnVCBERABERABERABERABERABERABERABNqbgESm9j5+6r0IiIAIiIAIiIAIiIAIiIAIiIAIiIAItAQBiUwtcRjUCREQAREQAREQAREQAREQAREQAREQARFobwISmdr7+Kn3IiACIiACIiACIiACIiACIiACIiACItASBCQytcRhUCdEQAREQAREQAREQAREQAREQAREQAREoL0JSGRq7+On3ouACIiACIiACIiACIiACIiACIiACIhASxD4P9UJI2/Ixb1iAAAAAElFTkSuQmCC", + "text/plain": [ + "" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import Image, SVG\n", + "Image(filename='../../../docs/source/_figures/remote_2.png')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As seen here, the GraphStore is used to store the neighbor relations between the nodes of the graph, whereas the FeatureStore is used to store the node and edge features in the graph." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start by loading in a knowledge graph dataset for the sake of our experiment:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import LargeGraphIndexer\n", + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", + "from itertools import chain" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Limiting to 10 questions for the sake of compute, but can be increased if necessary\n", + "ds = UpdatedWebQSPDataset(root='demo', limit=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's set up our set of questions and graph triplets:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['what is the name of justin bieber brother',\n", + " 'what character did natalie portman play in star wars',\n", + " 'what country is the grand bahama island in',\n", + " 'what kind of money to take to bahamas',\n", + " 'what character did john noble play in lord of the rings',\n", + " 'who does joakim noah play for',\n", + " 'where are the nfl redskins from',\n", + " 'where did saki live',\n", + " 'who did draco malloy end up marrying',\n", + " 'which countries border the us']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "questions = ds.raw_dataset['question']\n", + "questions" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", + " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", + " ['Rudolph Valentino',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", + " ['Record producer',\n", + " 'music.performance_role.regular_performances',\n", + " 'm.012m1vf1'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", + " ['2011 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0yrkr34'],\n", + " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Stratford', 'location.location.containedby', 'Canada'],\n", + " ['Singer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Musicians and Singers'],\n", + " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", + " ['As Long As You Love Me (Audien dubstep mix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kevin Risto', 'people.person.gender', 'Male'],\n", + " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", + " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", + " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", + " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", + " ['American Music Award for Favorite Pop/Rock Album',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc259'],\n", + " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", + " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", + " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Billboard Music Award for Top Streaming Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0njhx1b'],\n", + " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", + " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", + " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", + " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Results Show: Week 7'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['One Less Lonely Girl',\n", + " 'music.album.primary_release',\n", + " 'One Less Lonely Girl'],\n", + " ['Twista', 'people.person.gender', 'Male'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", + " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", + " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", + " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", + " ['Madonna', 'music.artist.genre', 'Pop music'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", + " ['m.0gbm3cg',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", + " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", + " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", + " ['MTV Video Music Award Japan for Best New Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhrwc'],\n", + " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", + " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", + " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['m.0gctwjk',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Series 2, Episode 3'],\n", + " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", + " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", + " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", + " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", + " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Beauty and a Beat',\n", + " 'music.single.versions',\n", + " 'Beauty and a Beat (Wideboys Club Mix)'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Bryan Adams',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male'],\n", + " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", + " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", + " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'music.composition.form', 'Song'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0yrkr34'],\n", + " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", + " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", + " ['Record producer',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Haley James Scott'],\n", + " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", + " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['Kanye West', 'people.person.profession', 'Singer'],\n", + " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", + " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", + " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", + " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", + " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", + " ['RedOne', 'music.artist.genre', 'Rock music'],\n", + " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", + " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", + " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'music.recording.featured_artists',\n", + " 'Big Sean'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", + " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", + " ['Athan Grace', 'people.person.profession', 'Actor'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", + " ['All Around the World (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", + " ['Brandy Norwood',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Whitney Houston'],\n", + " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['MTV Video Music Award for Artist to Watch',\n", + " 'award.award_category.winners',\n", + " 'm.0n1ykxp'],\n", + " ['Caitlin Beadles',\n", + " 'celebrities.celebrity.sexual_relationships',\n", + " 'm.0d33gyj'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audiobot instrumental)'],\n", + " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", + " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'The Mighty Mighty Bosstones'],\n", + " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", + " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", + " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Documentary film'],\n", + " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", + " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Amerie', 'music.artist.genre', 'Pop music'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", + " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep edit)'],\n", + " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Recovery', 'music.recording.song', 'Recovery'],\n", + " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", + " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", + " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", + " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", + " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", + " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", + " ['#thatPower', 'music.recording.song', '#thatPower'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", + " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['2013 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0wjgqck'],\n", + " ['The Island Def Jam Music Group',\n", + " 'organization.organization.parent',\n", + " 'm.04q65lb'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Rusted Root'],\n", + " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['m.0z87d3n',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", + " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", + " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", + " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", + " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", + " ['Tampa', 'location.location.containedby', 'United States of America'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.compositions',\n", + " 'Love Never Felt So Good'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", + " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", + " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", + " ['Estelle', 'people.person.profession', 'Record producer'],\n", + " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", + " ['Chris Jasper', 'people.person.gender', 'Male'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", + " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", + " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", + " ['Snoop Dogg',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['David Nicksay', 'people.person.gender', 'Male'],\n", + " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", + " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Juno Awards of 2014',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0102z0vx'],\n", + " ['As Long As You Love Me (Audiobot remix)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", + " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", + " ['m.0_cyzs_',\n", + " 'celebrities.legal_entanglement.offense',\n", + " 'Driving under the influence'],\n", + " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", + " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", + " ['Mann', 'people.person.gender', 'Male'],\n", + " ['JoJo', 'people.person.gender', 'Female'],\n", + " ['Right Here (featuring Drake)',\n", + " 'music.recording.canonical_version',\n", + " 'Right Here'],\n", + " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", + " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0yrjynf',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", + " ['Pras', 'people.person.profession', 'Record producer'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", + " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", + " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", + " ['Marvin Isley',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['School Gyrls', 'film.film.language', 'English Language'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", + " ['Katy Perry', 'people.person.profession', 'Actor'],\n", + " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", + " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.featured_artists',\n", + " 'Redfoo'],\n", + " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", + " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", + " ['As Long as You Love Me (album version)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Justin Bieber: Just Getting Started',\n", + " 'book.written_work.author',\n", + " 'Justin Bieber'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Marc Maris vs. Ramone'],\n", + " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", + " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", + " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", + " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.releases',\n", + " 'Love Never Felt So Good'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", + " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", + " ['Dance music',\n", + " 'broadcast.genre.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", + " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", + " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", + " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", + " ['Scooter Braun', 'people.person.gender', 'Male'],\n", + " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", + " ['Sir Nolan', 'people.person.gender', 'Male'],\n", + " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", + " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", + " ['Adam Messinger',\n", + " 'music.composer.compositions',\n", + " \"Turn to You (Mother's Day Dedication)\"],\n", + " ['m.0yrktlv',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Male Hottie'],\n", + " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", + " ['Iggy Azalea',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", + " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['m.0gxnnzy',\n", + " 'celebrities.romantic_relationship.relationship_type',\n", + " 'Dated'],\n", + " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", + " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", + " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Britney Spears',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", + " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", + " ['Fergie', 'music.artist.genre', 'Rock music'],\n", + " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", + " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", + " ['Mistletoe', 'music.album.release_type', 'Single'],\n", + " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", + " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.tracks',\n", + " 'Live My Life (Party Rock remix)'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['m.0njw4z2',\n", + " 'award.award_honor.award',\n", + " 'MTV Europe Music Award for Best Male'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", + " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", + " ['m.0gbm3c3',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Miley Cyrus'],\n", + " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", + " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", + " ['JoJo', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Next to You', 'music.album.release_type', 'Single'],\n", + " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", + " ['Willa Ford', 'people.person.languages', 'English Language'],\n", + " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", + " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", + " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", + " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Hot Wired Radio',\n", + " 'broadcast.content.broadcast',\n", + " 'Hot Wired Radio - 128kbps Stream'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", + " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0gbm3d9',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", + " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", + " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", + " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", + " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", + " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", + " ['Justin Timberlake',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Khalil', 'people.person.gender', 'Male'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", + " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", + " ['Selena Gomez',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", + " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", + " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['C1', 'music.artist.genre', 'Hip hop music'],\n", + " ['My Worlds Acoustic',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Album content type'],\n", + " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", + " ['Live My Life', 'music.composition.language', 'English Language'],\n", + " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", + " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", + " ['Baby', 'music.album.releases', 'Baby'],\n", + " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", + " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", + " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", + " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", + " ['Big R Radio - The Hawk',\n", + " 'broadcast.content.artist',\n", + " 'The Black Eyed Peas'],\n", + " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", + " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", + " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", + " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.production_companies',\n", + " 'AEG Live'],\n", + " ['Redfoo', 'people.person.gender', 'Male'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", + " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", + " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", + " ['m.01053qzf',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", + " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", + " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", + " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", + " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", + " ['Ray J', 'people.person.profession', 'Musician'],\n", + " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", + " ['m.0101fv5f',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", + " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", + " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", + " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", + " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Guest host'],\n", + " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", + " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", + " ['Christina Milian', 'people.person.profession', 'Actor'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", + " ['Dance music', 'broadcast.genre.content', '181-party'],\n", + " ['Queen Elizabeth II Diamond Jubilee Medal',\n", + " 'award.award_category.winners',\n", + " 'm.0njwqrb'],\n", + " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", + " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", + " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['CMT Music Award: Collaborative Video of the Year',\n", + " 'award.award_category.winners',\n", + " 'm.0njvs9s'],\n", + " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", + " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", + " ['Chingy', 'people.person.profession', 'Actor'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", + " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", + " ['Justin Bieber',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", + " ['Chingy', 'people.person.gender', 'Male'],\n", + " ['Reed Smoot', 'people.person.gender', 'Male'],\n", + " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", + " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.releases',\n", + " 'As Long As You Love Me (remixes)'],\n", + " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjvlh'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", + " ['Singer', 'common.topic.article', 'm.09l6h'],\n", + " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", + " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0pc670l'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", + " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", + " ['Beyoncé Knowles',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['m.0njhyh_',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Streaming Song (Video)'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", + " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_production_design_by',\n", + " 'Devorah Herbert'],\n", + " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", + " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", + " ['Nick Jonas',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", + " ['Lupe Fiasco',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['Martin Kierszenbaum',\n", + " 'people.person.place_of_birth',\n", + " 'United States of America'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long as You Love Me'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", + " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", + " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", + " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", + " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", + " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0y5tl39',\n", + " 'film.personal_film_appearance.film',\n", + " 'Les Coulisses des Golden Globes'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", + " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z0tgz6'],\n", + " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", + " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", + " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", + " ['Everlast', 'music.artist.label', 'Island Records'],\n", + " ['5th Annual Shorty Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0ywvh8k'],\n", + " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Baby', 'common.topic.notable_types', 'Composition'],\n", + " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['m.0tkqqgg',\n", + " 'award.award_nomination.award',\n", + " 'Juno Award for Pop Album of the Year'],\n", + " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Nasri', 'people.person.profession', 'Singer'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.compositions',\n", + " 'As Long as You Love Me'],\n", + " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", + " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", + " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", + " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", + " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", + " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", + " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", + " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", + " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", + " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", + " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.film',\n", + " 'Zendaya: Behind the Scenes'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", + " ['That Should Be Me', 'music.composition.form', 'Song'],\n", + " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", + " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Justin Bieber'],\n", + " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.song',\n", + " 'Beauty And A Beat'],\n", + " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", + " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Usher', 'music.composer.compositions', 'First Dance'],\n", + " ['m.0n1ykxp',\n", + " 'award.award_honor.award',\n", + " 'MTV Video Music Award for Artist to Watch'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Rockumentary'],\n", + " ['Amerie', 'people.person.gender', 'Female'],\n", + " ['Real Change: Artists for Education',\n", + " 'film.film.personal_appearances',\n", + " 'm.0y5th3r'],\n", + " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", + " ['Beautiful and the Beat',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['PYD', 'common.topic.notable_types', 'Composition'],\n", + " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", + " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", + " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", + " ['iJustine', 'people.person.gender', 'Female'],\n", + " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", + " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", + " ['m.0ng_vkd',\n", + " 'film.personal_film_appearance.film',\n", + " 'A Michael Bublé Christmas'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", + " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", + " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", + " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Vanessa Hudgens',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", + " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", + " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Madonna', 'people.person.profession', 'Record producer'],\n", + " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", + " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", + " ['As Long as You Love Me (acoustic version)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", + " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", + " ['Rob Thomas', 'people.person.gender', 'Male'],\n", + " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", + " ['m.0hvlt03',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", + " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_srv2b'],\n", + " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", + " ['Donna Summer',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0101fszs',\n", + " 'film.personal_film_appearance.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Alanis Morissette',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Official website'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Jenna Andrews'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", + " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Ray J', 'people.person.nationality', 'United States of America'],\n", + " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Nelly Furtado',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['m.0gbm3fj',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", + " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", + " ['As Long As You Love Me (Ferry Corsten club dub)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", + " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", + " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", + " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", + " ['Guest host',\n", + " 'tv.non_character_role.tv_guest_personal_appearances',\n", + " 'm.0v_98y7'],\n", + " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", + " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", + " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", + " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", + " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", + " ['L.A. Reid', 'people.person.gender', 'Male'],\n", + " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['London', 'location.location.containedby', 'Ontario'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_set_decoration_by',\n", + " 'Lia Roldan'],\n", + " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", + " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Change Me', 'music.album.releases', 'Change Me'],\n", + " ['RedOne', 'music.artist.label', 'Island Records'],\n", + " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", + " ['All Around the World',\n", + " 'music.recording.canonical_version',\n", + " 'All Around the World'],\n", + " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0wjhc6c'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", + " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", + " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Lupe Fiasco'],\n", + " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", + " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", + " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", + " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", + " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Beauty and a Beat (Wideboys Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", + " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", + " ['m.0v90skf',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Male Artist'],\n", + " ['Ludacris', 'people.person.profession', 'Actor'],\n", + " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", + " ['Cameo appearance',\n", + " 'tv.special_tv_performance_type.episode_performances',\n", + " 'm.0v1lwt2'],\n", + " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", + " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'One Chance'],\n", + " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", + " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", + " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", + " ['Roller Coaster',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_x4zg3'],\n", + " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", + " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", + " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", + " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", + " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", + " ['m.0sgk_cw',\n", + " 'award.award_honor.award',\n", + " \"Kids' Choice Award for Favorite Song\"],\n", + " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", + " ['MTV Video Music Brazil Award for Best International Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhhqv'],\n", + " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", + " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", + " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", + " ['Britney Spears', 'people.person.profession', 'Singer'],\n", + " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", + " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Rock music',\n", + " 'base.webvideo.internet_video_genre.series',\n", + " 'Biscuithands, The Animated Musical'],\n", + " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", + " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Barry Weiss'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['MTV Europe Music Award for Best Male',\n", + " 'award.award_category.winners',\n", + " 'm.0z1scxk'],\n", + " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", + " ['Will Smith', 'people.person.profession', 'Actor'],\n", + " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", + " ['Will i Am',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Fabian', 'people.person.gender', 'Male'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", + " ['Somebody to Love (remix)',\n", + " 'music.album.primary_release',\n", + " 'Somebody to Love (remix)'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", + " ['Spouse', 'type.property.expected_type', 'Person'],\n", + " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", + " ['Baby', 'music.recording.artist', 'Ludacris'],\n", + " ['Rudolph Valentino',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", + " ['Judy Garland',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Kelly Clarkson',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'm.011h9_22'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", + " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", + " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0vp8rhw',\n", + " 'music.recording_contribution.album',\n", + " 'Beauty and a Beat (Remixes)'],\n", + " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", + " ['Katy Perry: Part of Me',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['m.0jsmvv5',\n", + " 'film.film_regional_release_date.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", + " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", + " ['All Around The World (featuring Ludacris)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", + " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", + " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", + " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", + " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", + " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep mix)'],\n", + " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Fergie', 'people.person.profession', 'Actor'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", + " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", + " ['Zac Efron', 'people.person.gender', 'Male'],\n", + " ['P!nk', 'music.artist.genre', 'Rock music'],\n", + " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", + " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", + " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", + " ['Under the Mistletoe',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Slick Rick'],\n", + " ['Amerie', 'music.artist.genre', 'Rock music'],\n", + " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0pbzq13',\n", + " 'film.performance.special_performance_type',\n", + " 'Cameo appearance'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", + " ['Height', 'type.property.unit', 'Meter'],\n", + " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", + " ['Ray J',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Boyfriend (acoustic version)',\n", + " 'music.recording.canonical_version',\n", + " 'Boyfriend'],\n", + " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Believe Tour',\n", + " 'music.concert_tour.album_or_release_supporting',\n", + " 'Believe'],\n", + " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", + " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", + " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", + " ['Frank Ocean',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['As Long As You Love Me (Audiobot instrumental)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Usher', 'people.person.nationality', 'United States of America'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", + " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", + " ['m.0gbm3b7',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Sheryl Crow',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['All That Matters',\n", + " 'music.composition.composer',\n", + " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Nasri', 'music.artist.genre', 'Reggae'],\n", + " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", + " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", + " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", + " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", + " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", + " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", + " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long As You Love Me'],\n", + " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", + " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", + " ['Whitney Houston', 'people.person.profession', 'Model'],\n", + " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['As Long as You Love Me',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Disney Parks Christmas Day Parade',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['Ray J', 'people.person.profession', 'Artist'],\n", + " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", + " ['American Music Award for Favorite Pop/Rock Male Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc0sf'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", + " ['The Notorious B.I.G.',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", + " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", + " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", + " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", + " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Haley James Scott',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Record producer'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", + " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", + " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['My Worlds: The Collection',\n", + " 'music.album.album_content_type',\n", + " 'Compilation album'],\n", + " ['Lolly', 'music.album.releases', 'Lolly'],\n", + " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", + " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", + " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", + " ['Ja Rule', 'people.person.profession', 'Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", + " ['Die in Your Arms',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z85qxq'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", + " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", + " ['Kuk Harrell',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0101ft5f'],\n", + " ['Somebody to Love (J Stax remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'film.film.executive_produced_by',\n", + " 'Allison Kaye Scarinzi'],\n", + " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", + " ['Nasri', 'music.artist.genre', 'Pop music'],\n", + " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", + " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", + " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.05sp405',\n", + " 'organization.organization_relationship.child',\n", + " 'Island Records'],\n", + " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", + " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", + " ['John Mamann',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Teen Choice Award for Choice Summer Music Star: Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjynf'],\n", + " ['Juicy J', 'people.person.profession', 'Actor'],\n", + " ['m.0yqflrk',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'celebritynetworth.com'],\n", + " ['Miley Cyrus',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", + " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['m.04q65lb',\n", + " 'organization.organization_relationship.child',\n", + " 'The Island Def Jam Music Group'],\n", + " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", + " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", + " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['My World', 'music.album.artist', 'Justin Bieber'],\n", + " ['Don Henley', 'people.person.gender', 'Male'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Musician', 'people.profession.specializations', 'Singer'],\n", + " ['Die in Your Arms',\n", + " 'music.recording.canonical_version',\n", + " 'Die in Your Arms'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['m.0njvs9s',\n", + " 'award.award_honor.award',\n", + " 'CMT Music Award: Collaborative Video of the Year'],\n", + " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber',\n", + " 'music.artist.album',\n", + " 'Turn to You (Mother’s Day Dedication)'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", + " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", + " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Terence Dudley', 'people.person.gender', 'Male'],\n", + " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Mr. Sosa & The Yayo'],\n", + " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", + " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", + " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0101fvbf',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['m.0yrhrwc',\n", + " 'award.award_honor.ceremony',\n", + " '2011 MTV Video Music Aid Japan'],\n", + " ['MTV Europe Music Award for Best North American Act',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhmll'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['m.0101ft1d',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", + " ['Ciara', 'people.person.gender', 'Female'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", + " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", + " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0jzrrqs',\n", + " 'location.mailing_address.country',\n", + " 'United States of America'],\n", + " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", + " ['Big R Radio - Top 40 Hits',\n", + " 'broadcast.content.artist',\n", + " 'Natasha Bedingfield'],\n", + " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ...],\n", + " [['m.0t4tf9z', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Jonathan Glickman',\n", + " 'film.producer.films_executive_produced',\n", + " 'No Strings Attached'],\n", + " ['Anywhere but Here', 'film.film.language', 'English Language'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Shiri Appleby'],\n", + " ['m.0g4xwxr',\n", + " 'film.film_crew_gig.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Veganism', 'religion.religious_practice.practice_of', 'Jainism'],\n", + " ['m.0w9tk0s', 'tv.tv_guest_personal_appearance.person', 'Natalie Portman'],\n", + " ['m.0nfwmy8', 'award.award_nomination.award', 'SFX Award for Best Actress'],\n", + " ['Hesher', 'film.film.directed_by', 'Spencer Susser'],\n", + " ['Zac Posen', 'base.popstra.company.fashion_choice', 'm.077jny5'],\n", + " ['Bar Refaeli', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['New York, I Love You', 'film.film.country', 'United States of America'],\n", + " ['m.0w9tkgh', 'tv.tv_guest_personal_appearance.person', 'Natalie Portman'],\n", + " ['Israelis', 'common.topic.notable_for', 'g.1255cyblx'],\n", + " ['m.0w4f0kn',\n", + " 'award.award_honor.award',\n", + " 'Kansas City Film Critics Circle Award for Best Actress'],\n", + " ['Arnon Milchan', 'common.topic.notable_types', 'Film producer'],\n", + " ['Yonatan Netanyahu',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['The Weathered Underground', 'common.topic.notable_types', 'Film'],\n", + " ['New York, I Love You', 'film.film.directed_by', 'Yvan Attal'],\n", + " ['Hesher', 'film.film.release_date_s', 'm.0qsjltl'],\n", + " ['Beautiful Girls', 'film.film.genre', 'Comedy'],\n", + " ['m.0y627h9', 'film.performance.film', 'No Strings Attached'],\n", + " ['Where the Heart Is', 'media_common.netflix_title.netflix_genres', 'Drama'],\n", + " ['Anywhere but Here', 'common.topic.notable_types', 'Film'],\n", + " ['Eve', 'film.film.starring', 'm.05cgmyb'],\n", + " ['m.0k3qzz', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Cold Mountain', 'freebase.valuenotation.is_reviewed', 'Directed by'],\n", + " ['m.0cp8_d4', 'film.performance.character', 'Emma'],\n", + " ['m.0qsjnjf',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'United States of America'],\n", + " ['m.07zq3__',\n", + " 'award.award_nomination.award',\n", + " 'BAFTA Award for Best Actress in a Supporting Role'],\n", + " ['Ayelet Waldman', 'people.person.languages', 'English Language'],\n", + " ['m.0vnx4f9', 'film.performance.film', 'Hesher'],\n", + " ['m.0w4dzvx', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Dramas Based on Contemporary Literature'],\n", + " ['Gene Simmons', 'people.person.profession', 'Film Producer'],\n", + " ['Remi Adefarasin', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Yitzhak Navon', 'people.person.place_of_birth', 'Jerusalem'],\n", + " ['Isaac Mizrahi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Film producer', 'freebase.type_profile.strict_included_types', 'Person'],\n", + " ['Saar Klein', 'people.person.gender', 'Male'],\n", + " ['Hesher', 'film.film.runtime', 'm.0h9hbwk'],\n", + " ['m.0fpy598',\n", + " 'award.award_nomination.ceremony',\n", + " \"16th Critics' Choice Awards\"],\n", + " ['Natalie Portman', 'base.popstra.celebrity.dated', 'm.065q1bw'],\n", + " ['Chaim Weizmann', 'people.person.religion', 'Judaism'],\n", + " ['Eve', 'film.film.directed_by', 'Natalie Portman'],\n", + " ['m.0k0_f3', 'film.performance.film', 'My Blueberry Nights'],\n", + " ['Thor: The Dark World', 'film.film.starring', 'm.0jrhb6s'],\n", + " ['m.0w4f166',\n", + " 'award.award_honor.award',\n", + " 'Dallas–Fort Worth Film Critics Association Award for Best Actress'],\n", + " ['Semitic people', 'people.ethnicity.languages_spoken', 'Hebrew Language'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tj1c'],\n", + " ['m.0zdv41v', 'film.film_crew_gig.film', 'Jane Got a Gun'],\n", + " ['Zac Posen', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.release_date_s',\n", + " 'm.0g4xqxm'],\n", + " ['Molly Mahoney', 'film.film_character.portrayed_in_films', 'm.0k0x4n'],\n", + " ['Jewish people', 'people.ethnicity.languages_spoken', 'Yiddish Language'],\n", + " ['No Strings Attached', 'film.film.release_date_s', 'm.0j57m4x'],\n", + " ['Natalie Portman',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Hesher', 'film.film.release_date_s', 'm.0qsk1b6'],\n", + " ['m.0cp8_fg', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0z9wm_1', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['m.0w9tjfg',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Film Producer',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010b13c2'],\n", + " ['The Other Boleyn Girl',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Directed by'],\n", + " ['m.0z9c492', 'freebase.valuenotation.has_no_value', 'Nominated work'],\n", + " ['m.0w9tkdv', 'tv.tv_guest_personal_appearance.episode', 'Natalie Portman'],\n", + " [\"NATALIE PORTMAN, BILL O'REILLY, PATTY LOVELESS\",\n", + " 'common.topic.notable_types',\n", + " 'TV Episode'],\n", + " ['Ehud Barak', 'people.person.ethnicity', 'Israelis'],\n", + " ['Yitzhak Ben-Zvi', 'people.deceased_person.place_of_death', 'Jerusalem'],\n", + " ['Bar Refaeli', 'people.person.gender', 'Female'],\n", + " ['Jane Got a Gun', 'film.film.country', 'United States of America'],\n", + " ['Las Vegas Film Critics Society Awards 2010',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0w4f079'],\n", + " ['David Michôd', 'people.person.nationality', 'United States of America'],\n", + " ['Scot Armstrong', 'people.person.gender', 'Male'],\n", + " ['Everyone Says I Love You',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['m.0pdhvh0',\n", + " 'award.award_nomination.award',\n", + " 'Razzie Award for Worst Screen Couple/Ensemble'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.0131yzfn'],\n", + " ['Avi Lerner', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Big Sur', 'common.topic.notable_types', 'Location'],\n", + " ['m.0y60_p3', 'film.performance.film', 'No Strings Attached'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0w4f1jr'],\n", + " ['m.0z9c492',\n", + " 'award.award_nomination.award',\n", + " 'Teen Choice Award for Choice Activist'],\n", + " ['Screenwriter', 'common.topic.subject_of', 'Lew M. Parry'],\n", + " ['Anywhere but Here',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Romantic Movies'],\n", + " [\"Natalie's Rap\", 'music.recording.tracks', 'Natalie’s Rap'],\n", + " ['Yvan Attal', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0b6rk43', 'common.webpage.topic', 'Veganism'],\n", + " [\"Goya's Ghosts\", 'film.film.starring', 'm.0jyzhq'],\n", + " ['Devendra Banhart',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['m.0w4dz_y', 'award.award_honor.ceremony', '37th Saturn Awards'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Rahm Emanuel', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0gm46pt'],\n", + " ['Screen Actors Guild Award for Outstanding Performance by a Female Actor in a Leading Role',\n", + " 'award.award_category.winners',\n", + " 'm.0p75gts'],\n", + " ['Garden State', 'film.film.language', 'English Language'],\n", + " ['m.065pj6n', 'base.popstra.friendship.participant', 'Julia Roberts'],\n", + " ['m.0w9tjyf',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Don Roos', 'people.person.nationality', 'United States of America'],\n", + " ['Kim Davis', 'film.film_casting_director.films_casting_directed', 'Hesher'],\n", + " ['m.05cgmwh', 'film.film_cut.film', 'Eve'],\n", + " ['Pride and Prejudice and Zombies', 'common.topic.webpage', 'm.0dk5x74'],\n", + " ['Ralph Bakshi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.065pt8q', 'base.popstra.dated.participant', 'Jake Gyllenhaal'],\n", + " ['23rd Golden Raspberry Awards',\n", + " 'award.award_ceremony.nominees',\n", + " 'm.0pdhvh0'],\n", + " ['m.0h37dxx', 'base.popstra.friendship.participant', 'Bryce Dallas Howard'],\n", + " ['Hesher', 'film.film.starring', 'm.0gcy8zl'],\n", + " ['Natalie Portman', 'people.person.education', 'm.0n0l04n'],\n", + " ['m.0z9tsh9', 'award.award_nomination.nominated_for', 'Garden State'],\n", + " ['Natalie Portman', 'film.actor.film', 'm.0gx90ts'],\n", + " ['m.0w9tk6b',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Star Wars Episode III: Revenge of the Sith',\n", + " 'film.film.genre',\n", + " 'Action Film'],\n", + " ['Javier Bardem', 'base.popstra.celebrity.friendship', 'm.065pl8n'],\n", + " ['Peter Fruchtman', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Meir Kahane'],\n", + " ['Gael García Bernal', 'people.person.profession', 'Actor'],\n", + " ['m.099fg92', 'film.performance.film', 'Hesher'],\n", + " ['Benjamin Millepied', 'people.person.spouse_s', 'm.0gx8_vd'],\n", + " ['Developing', 'film.film.starring', 'm.0gx90ts'],\n", + " ['Menachem Begin', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Jane Foster', 'film.film_character.portrayed_in_films', 'm.0855mj_'],\n", + " ['Ethnicity', 'freebase.type_profile.published', 'Published'],\n", + " ['Lisa Gerrard', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Ariel Sharon', 'people.person.religion', 'Judaism'],\n", + " ['m.0fpnkn8',\n", + " 'award.award_nomination.award',\n", + " 'Satellite Award for Best Actress – Motion Picture Drama'],\n", + " ['Yitzhak Rabin', 'people.person.nationality', 'Israel'],\n", + " ['Emmanuel Benbihy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0pc689n',\n", + " 'award.award_nomination.award',\n", + " 'MTV Movie Award for Best Kiss'],\n", + " ['Show #2278', 'common.topic.notable_types', 'TV Episode'],\n", + " ['Veganism', 'common.topic.article', 'm.07_j6'],\n", + " ['Teen Choice Award for Choice Movie Actress',\n", + " 'award.award_category.winners',\n", + " 'm.0w4dyr8'],\n", + " ['m.059phw1',\n", + " 'award.award_nomination.ceremony',\n", + " '23rd Golden Raspberry Awards'],\n", + " ['m.0cp8_hb', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0j2f88v', 'film.film_cut.film', 'Love and Other Impossible Pursuits'],\n", + " ['m.0k0gpv', 'film.performance.character', 'Taffy Dale'],\n", + " ['m.0w9tjyf', 'tv.tv_guest_personal_appearance.person', 'Natalie Portman'],\n", + " ['m.03jt5jq', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Roger Birnbaum', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Oded Fehr', 'people.person.ethnicity', 'Israeli American'],\n", + " ['Roger Birnbaum', 'people.person.profession', 'Film Producer'],\n", + " ['Anthony Edwards', 'base.popstra.celebrity.friendship', 'm.065pktf'],\n", + " ['Brothers', 'media_common.netflix_title.netflix_genres', 'Drama'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['m.02wmw6l', 'education.education.institution', 'Harvard College'],\n", + " ['New York, I Love You', 'common.topic.notable_types', 'Film'],\n", + " ['Moshe Katsav', 'people.person.religion', 'Judaism'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tktt'],\n", + " ['m.0gkkdpm', 'film.performance.film', 'Hesher'],\n", + " ['m.0w4dyr8', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['m.0w0wbjd', 'film.personal_film_appearance.person', 'Natalie Portman'],\n", + " ['Free Zone', 'common.topic.notable_types', 'Film'],\n", + " ['Where the Heart Is',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z9x3f_'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y6235s'],\n", + " ['m.0w4f166',\n", + " 'award.award_honor.ceremony',\n", + " 'Dallas–Fort Worth Film Critics Association Awards 2010'],\n", + " ['Zoolander', 'film.film.language', 'English Language'],\n", + " ['m.0y62f8k', 'film.performance.film', 'No Strings Attached'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tk56'],\n", + " ['m.02wmw6l', 'education.education.major_field_of_study', 'Psychology'],\n", + " ['Ran Danker', 'people.person.nationality', 'Israel'],\n", + " ['m.0fq7b0k',\n", + " 'award.award_nomination.award',\n", + " 'Screen Actors Guild Award for Outstanding Performance by a Cast in a Motion Picture'],\n", + " ['Joe Medjuck', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Anywhere but Here',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Dramas based on contemporary literature'],\n", + " ['Israelis', 'people.ethnicity.people', 'Yitzhak HaLevi Herzog'],\n", + " ['BAFTA Award for Best Actress in a Leading Role',\n", + " 'award.award_category.winners',\n", + " 'm.0gbm5y_'],\n", + " ['Gael García Bernal',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Spouse (or domestic partner)'],\n", + " ['2002 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0z9wmn8'],\n", + " ['Closer', 'award.award_nominated_work.award_nominations', 'm.09tf43b'],\n", + " ['Cold Mountain', 'film.film.rating', 'R (USA)'],\n", + " [\"10th Critics' Choice Awards\",\n", + " 'award.award_ceremony.nominees',\n", + " 'm.09zl530'],\n", + " ['m.065q38y', 'base.popstra.dated.participant', 'Natalie Portman'],\n", + " ['Free Zone', 'film.film.genre', 'Comedy'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0w4dz_y'],\n", + " ['m.0fq7b0k', 'award.award_nomination.award_nominee', 'Vincent Cassel'],\n", + " ['Developing', 'film.film.country', 'United States of America'],\n", + " ['Robert Aumann', 'people.person.nationality', 'United States of America'],\n", + " ['No Strings Attached', 'film.film.other_crew', 'm.0g4ndyt'],\n", + " ['Peter Fruchtman', 'people.person.profession', 'Film Producer'],\n", + " ['Zalman Shazar', 'people.person.nationality', 'Israel'],\n", + " ['Wholphin: Issue 10', 'film.film.starring', 'm.0gxgdlj'],\n", + " ['Clive Owen', 'people.person.profession', 'Film Producer'],\n", + " ['David Michôd', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Peter Fruchtman',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0y60z23', 'film.performance.film', 'No Strings Attached'],\n", + " ['Natalie Portman/Fall Out Boy',\n", + " 'base.saturdaynightlive.snl_episode.host',\n", + " 'Natalie Portman'],\n", + " ['No Strings Attached',\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'g.125_pty8h'],\n", + " ['Black Swan', 'award.award_winning_work.awards_won', 'm.0w4d_56'],\n", + " ['Arnon Milchan', 'people.person.gender', 'Male'],\n", + " ['Film Producer',\n", + " 'award.award_discipline.awards_in_this_discipline',\n", + " 'Irving G. Thalberg Memorial Award'],\n", + " ['m.0w9tk56',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'The 77th Annual Academy Awards'],\n", + " ['m.04687mg',\n", + " 'award.award_nomination.award',\n", + " 'Saturn Award for Best Actress'],\n", + " ['m.0g4nfq7', 'film.performance.film', 'No Strings Attached'],\n", + " ['No Strings Attached', 'film.film.film_casting_director', 'Joanna Colbert'],\n", + " [\"Goya's Ghosts\", 'common.topic.notable_types', 'Film'],\n", + " ['m.0zdv4mz', 'film.film_crew_gig.film', 'Jane Got a Gun'],\n", + " ['Israelis', 'people.ethnicity.people', 'Meir Dagan'],\n", + " ['m.0y62321', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.05c79pt', 'award.award_nomination.ceremony', '77th Academy Awards'],\n", + " ['m.0w4f0kn', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['Shelley Stevens', 'people.person.profession', 'Talent agent'],\n", + " ['Devendra Banhart', 'people.person.profession', 'Artist'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0t4td2m'],\n", + " ['Mike Nichols',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Hesher', 'film.film.film_set_decoration_by', 'Jennifer Lukehart'],\n", + " ['Mike Nichols', 'people.person.ethnicity', 'Jewish people'],\n", + " ['m.0y62gmy', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0w9tktt', 'tv.tv_guest_personal_appearance.episode', 'Terry McAuliffe'],\n", + " ['Thor: The Dark World',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Film Producer',\n", + " 'award.award_discipline.awards_in_this_discipline',\n", + " 'David di Donatello for Best Producer'],\n", + " ['Leah Goldberg', 'people.person.languages', 'English Language'],\n", + " ['Celebrity', 'type.type.properties', 'Celebrity rivals'],\n", + " ['m.0w9tk6b', 'tv.tv_guest_personal_appearance.person', 'Natalie Portman'],\n", + " ['17th Screen Actors Guild Awards',\n", + " 'award.award_ceremony.nominees',\n", + " 'm.0fq78zw'],\n", + " ['New York, I Love You',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Romantic Movies'],\n", + " ['Dan Bucatinsky', 'people.person.nationality', 'United States of America'],\n", + " ['m.0nfwhqy', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['m.0t4tc_t', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['m.012zh2mn', 'film.performance.film', 'Pride and Prejudice and Zombies'],\n", + " ['Brothers', 'film.film.rating', 'R (USA)'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0n2b8hz'],\n", + " ['Omri Katz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Star Wars Episode II: Attack of the Clones',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z9wqpt'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0t4tddr'],\n", + " ['Natalie Portman',\n", + " 'book.author.works_written',\n", + " 'Frontal Lobe Activation during Object Permanence: Data from Near-Infrared Spectroscopy'],\n", + " ['Bryce Dallas Howard',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.0w9tk0s',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Year', 'rdf-schema#range', 'Date/Time'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9thjt'],\n", + " ['Tom Pollock', 'people.person.gender', 'Male'],\n", + " ['Israelis', 'people.ethnicity.people', 'Bar Refaeli'],\n", + " ['m.0gkm43n', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['m.0gm46pt',\n", + " 'award.award_nomination.award',\n", + " 'National Board of Review Award for Best Cast'],\n", + " ['Rebecca', 'film.film_character.portrayed_in_films', 'm.0jygjj'],\n", + " ['Hesher', 'film.film.release_date_s', 'm.0qsk1s3'],\n", + " ['Burr Steers', 'people.person.place_of_birth', 'Washington, D.C.'],\n", + " ['Drama', 'media_common.media_genre.child_genres', 'Romance Film'],\n", + " ['Ehud Olmert', 'people.person.ethnicity', 'Israelis'],\n", + " ['The Weathered Underground', 'film.film.genre', 'Comedy'],\n", + " ['Javier Bardem', 'people.person.gender', 'Male'],\n", + " ['m.0nfnhrj', 'film.performance.actor', 'Natalie Portman'],\n", + " ['m.040myw2', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Hesher', 'film.film.starring', 'm.0vn_pxc'],\n", + " ['Bar Refaeli', 'freebase.valuenotation.has_value', 'Siblings'],\n", + " ['The Other Boleyn Girl',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Romantic Movies'],\n", + " ['V for Vendetta', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['Ayelet Waldman',\n", + " 'book.author.works_written',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Spyglass Entertainment',\n", + " 'film.production_company.films',\n", + " 'No Strings Attached'],\n", + " ['m.0gxgdlj', 'film.performance.film', 'Wholphin: Issue 10'],\n", + " ['Screen Actors Guild Award for Outstanding Performance by a Cast in a Motion Picture',\n", + " 'award.award_category.nominees',\n", + " 'm.0fq7b0k'],\n", + " ['m.0vnxk3g', 'film.performance.film', 'Hesher'],\n", + " ['Zach Braff', 'award.award_nominee.award_nominations', 'm.0z9t9pr'],\n", + " ['Natalie Portman', 'celebrities.celebrity.sexual_orientation', 'm.04f9t5b'],\n", + " ['New York, I Love You', 'film.film.written_by', 'Natalie Portman'],\n", + " ['Natalie Portman', 'people.person.gender', 'Female'],\n", + " ['m.0gx8_v0', 'freebase.valuenotation.has_value', 'End Date'],\n", + " ['Israelis', 'people.ethnicity.people', 'Tova Milo'],\n", + " ['Star Wars Episode II: Attack of the Clones',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['The Weathered Underground',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Indie Comedies'],\n", + " ['Hesher', 'film.film.language', 'English Language'],\n", + " ['Siblings', 'rdf-schema#domain', 'Person'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y6clbd'],\n", + " ['m.0cp8zg5',\n", + " 'film.film_film_distributor_relationship.region',\n", + " 'United States of America'],\n", + " ['Israelis', 'people.ethnicity.people', 'Emile Habibi'],\n", + " ['Zoolander', 'media_common.netflix_title.netflix_genres', 'Comedies'],\n", + " ['Film Producer', 'common.topic.subjects', 'MA$TADON'],\n", + " ['m.04dcjy9', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Bar Refaeli', 'common.topic.notable_types', 'Celebrity'],\n", + " ['Meir Kahane', 'people.person.nationality', 'United States of America'],\n", + " ['Judaism', 'religion.religion.sacred_sites', 'Jerusalem'],\n", + " ['Avi Lerner', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Julia Roberts', 'common.topic.notable_types', 'Celebrity'],\n", + " ['m.0v1_0z0', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['Eve', 'common.topic.notable_types', 'Film'],\n", + " ['Natalie Portman', 'people.person.employment_history', 'm.07pmhpl'],\n", + " ['Winona Ryder',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Yvan Attal', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0gm46q7'],\n", + " ['Black Swan: Metamorphosis', 'common.topic.notable_types', 'Film'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y62240'],\n", + " [\"Milos Forman: What doesn't kill you\",\n", + " 'film.film.language',\n", + " 'English Language'],\n", + " ['Jude Law', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Black Swan', 'award.award_winning_work.awards_won', 'm.0w4dzvx'],\n", + " ['Iddo Netanyahu', 'people.person.gender', 'Male'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.country',\n", + " 'United States of America'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.012zfr09'],\n", + " ['m.0w4dzf8',\n", + " 'award.award_honor.ceremony',\n", + " 'Southeastern Film Critics Association Awards 2010'],\n", + " ['Gene Simmons', 'people.person.profession', 'Screenwriter'],\n", + " ['Veganism', 'common.topic.notable_types', 'Diet'],\n", + " ['Ralph Bakshi', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Zac Posen', 'people.person.gender', 'Male'],\n", + " ['Your Highness', 'film.film.genre', 'Comedy'],\n", + " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", + " ['Jude Law', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Israelis', 'people.ethnicity.people', 'Gila Katsav'],\n", + " ['Marco Beltrami',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.05cgmvz', 'film.performance.film', 'Eve'],\n", + " ['Hayden Christensen', 'base.popstra.celebrity.dated', 'm.065q7s8'],\n", + " ['Film Producer',\n", + " 'music.music_video_job.music_videos_with_this_crew_job',\n", + " 'm.011dhk15'],\n", + " ['Black Swan: Metamorphosis', 'film.film.personal_appearances', 'm.0h12kcc'],\n", + " ['m.0z9s_qp', 'award.award_nomination.nominated_for', 'Garden State'],\n", + " ['Show #1321', 'common.topic.notable_types', 'TV Episode'],\n", + " ['Hesher', 'film.film.starring', 'm.0gdnl42'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0h5pbyb'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Weightless', 'film.film.country', 'United States of America'],\n", + " ['Mike Nichols', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Tricia Cooke', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Barbara Hershey', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0k3r0b', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Eve', 'film.film.written_by', 'Natalie Portman'],\n", + " ['Boston Society of Film Critics Awards 2010',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0w4f1cj'],\n", + " ['Shiri Appleby', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Mathilda', 'film.film_character.portrayed_in_films', 'm.0j_xb7'],\n", + " ['m.0nfnjqd', 'tv.tv_guest_role.episodes_appeared_in', 'Moonshine River'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y6248k'],\n", + " ['Avner Hershlag', 'common.topic.article', 'm.07tgpl1'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0zdrzm2', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['m.0w9tjj2',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Ezer Weizman', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['Omri Katz', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0cp8_8z', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0g8mcmc', 'award.award_nomination.nominated_for', 'Black Swan'],\n", + " ['Ivan Reitman', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Yvan Attal',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Hesher', 'common.topic.article', 'm.099f1k5'],\n", + " ['Chris Parnell', 'music.artist.track', \"Natalie's Rap\"],\n", + " ['m.0h37dxx', 'base.popstra.friendship.participant', 'Natalie Portman'],\n", + " ['Omri Katz', 'people.person.ethnicity', 'Jewish people'],\n", + " [\"I'm Still Here\", 'film.film.genre', 'Comedy'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0gm475f'],\n", + " ['Aleph Portman-Millepied', 'common.topic.article', 'm.0j2khsq'],\n", + " ['Black Swan', 'award.award_winning_work.awards_won', 'm.0w4dz_y'],\n", + " ['Clive Owen', 'people.person.gender', 'Male'],\n", + " ['Mila Kunis', 'people.person.profession', 'Model'],\n", + " ['No Strings Attached', 'film.film.film_casting_director', 'Richard Mento'],\n", + " ['From', 'type.property.expected_type', 'Date/Time'],\n", + " ['26th Independent Spirit Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0p433rm'],\n", + " ['John Debney', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tkr_'],\n", + " ['Gael García Bernal', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Hesher', 'film.film.starring', 'm.0vnw_y_'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Featured Song'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0w4d_56'],\n", + " ['Marc E. Platt', 'people.person.profession', 'Film Producer'],\n", + " ['Benjamin Millepied', 'freebase.valuenotation.has_value', 'Siblings'],\n", + " ['Zalman Shazar', 'people.deceased_person.place_of_death', 'Jerusalem'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Ralph Bakshi'],\n", + " ['Spouse', 'type.property.expected_type', 'Person'],\n", + " ['m.09k3pxr', 'award.award_nomination.award_nominee', 'Julia Roberts'],\n", + " ['m.09sj40s', 'film.performance.film', 'Hesher'],\n", + " ['Zach Braff', 'film.director.film', 'Garden State'],\n", + " ['Emmanuel Benbihy', 'film.writer.film', 'New York, I Love You'],\n", + " ['Semitic people', 'people.ethnicity.includes_groups', 'Jewish people'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y615_l'],\n", + " ['Heat', 'media_common.netflix_title.netflix_genres', 'Thriller'],\n", + " ['m.0y6c5jq', 'film.performance.film', 'No Strings Attached'],\n", + " ['My Blueberry Nights',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Independent Dramas'],\n", + " ['Benjamin Millepied',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Celebrity', 'type.type.expected_by', 'Celebrity'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0v208yl'],\n", + " ['David Ben-Gurion', 'people.person.ethnicity', 'Israelis'],\n", + " ['m.0pc69cp', 'award.award_nomination.ceremony', '2011 MTV Movie Awards'],\n", + " ['Natalie Portman - TIFF2010 01',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'Natalie Portman'],\n", + " ['m.0z87s8m',\n", + " 'award.award_nomination.award',\n", + " 'Teen Choice Award for Choice Movie Actress'],\n", + " ['Hayden Christensen',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.05cgmw3', 'film.performance.film', 'Eve'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['m.0w9tjyf', 'tv.tv_guest_personal_appearance.episode', 'Show #2214'],\n", + " ['Julia Roberts', 'people.person.gender', 'Female'],\n", + " ['m.0w4d_56',\n", + " 'award.award_honor.ceremony',\n", + " 'Phoenix Film Critics Society Awards 2010'],\n", + " ['m.0w4f1cj',\n", + " 'award.award_honor.ceremony',\n", + " 'Boston Society of Film Critics Awards 2010'],\n", + " ['Don Roos', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['No Strings Attached',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z87srm'],\n", + " ['Natalie Portman', 'people.person.ethnicity', 'Jewish people'],\n", + " ['m.0t4tcwg', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Avner Hershlag', 'people.person.spouse_s', 'm.0j4jpz0'],\n", + " ['m.0v1__b1', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['Oded Fehr', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.090dzsw',\n", + " 'award.award_nomination.award',\n", + " 'Golden Globe Award for Best Supporting Actress – Motion Picture'],\n", + " ['m.0w0wbjd', 'film.personal_film_appearance.film', 'A Powerful Noise Live'],\n", + " ['Javier Bardem', 'base.popstra.celebrity.dated', 'm.065q38y'],\n", + " ['Isaac Mizrahi', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0z87_sd', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Andy Samberg', 'people.person.gender', 'Male'],\n", + " ['Black Swan', 'award.award_nominated_work.award_nominations', 'm.0pc689n'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0w4f0kn'],\n", + " ['V for Vendetta',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Tom Pollock',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Israelis', 'people.ethnicity.people', 'Ran Danker'],\n", + " ['Shimon Peres', 'people.person.gender', 'Male'],\n", + " ['Natalie Portman', 'base.popstra.celebrity.friendship', 'm.0h2mv72'],\n", + " ['Nina Sayers', 'film.film_character.portrayed_in_films', 'm.0cccx3m'],\n", + " ['m.065pf9h', 'base.popstra.friendship.participant', 'Natalie Portman'],\n", + " ['Inés', 'film.film_character.portrayed_in_films', 'm.0jyzhq'],\n", + " ['m.0_h8yyt',\n", + " 'award.award_honor.honored_for',\n", + " 'Star Wars Episode II: Attack of the Clones'],\n", + " ['m.09zl530', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Hayden Christensen',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.09zl530',\n", + " 'award.award_nomination.award',\n", + " \"Critics' Choice Movie Award for Best Supporting Actress\"],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0v1z4wj'],\n", + " ['True', 'film.film.starring', 'm.0gx90td'],\n", + " ['m.0w9tjm4',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Israelis', 'people.ethnicity.people', 'Shoshana Damari'],\n", + " ['Jane Foster', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Israelis', 'people.ethnicity.people', 'Amir Bar-Lev'],\n", + " ['Jeanine Lobell', 'people.person.nationality', 'United States of America'],\n", + " ['Net worth', 'type.property.schema', 'Celebrity'],\n", + " ['Celebrity', 'type.type.expected_by', 'Celebrity'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Israel', 'location.country.languages_spoken', 'English Language'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.012zkg5x'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Gene Simmons'],\n", + " ['m.0gm46n_',\n", + " 'award.award_nomination.award',\n", + " 'Teen Choice Award for Choice Movie Actress'],\n", + " ['m.0fq7b0k',\n", + " 'award.award_nomination.ceremony',\n", + " '17th Screen Actors Guild Awards'],\n", + " ['Chaim Weizmann', 'people.person.gender', 'Male'],\n", + " ['Free Zone', 'film.film.language', 'Arabic Language'],\n", + " ['Peter Fruchtman', 'people.person.nationality', 'United States of America'],\n", + " ['m.0z9wm_1',\n", + " 'award.award_nomination.award',\n", + " 'Teen Choice Award for Movies - Choice Actress: Drama/Action Adventure'],\n", + " ['Hesher', 'film.film.release_date_s', 'm.0qsk1yz'],\n", + " ['Oren Moverman', 'people.person.profession', 'Screenwriter'],\n", + " ['Tom Pollock', 'people.person.profession', 'Film Producer'],\n", + " ['m.0zdrybw', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.production_companies',\n", + " 'Impossible Pursuits, LLC.'],\n", + " ['Meir Dagan', 'people.person.nationality', 'Israel'],\n", + " ['Jude Law', 'base.popstra.celebrity.dated', 'm.06529v3'],\n", + " ['Oren Moverman', 'people.person.profession', 'Film Producer'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y6c5jq'],\n", + " ['Aleph Portman-Millepied',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Country of nationality'],\n", + " ['Israel Horovitz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Chris Parnell', 'freebase.valuenotation.has_value', 'Siblings'],\n", + " ['m.0w4f079',\n", + " 'award.award_honor.award',\n", + " 'Las Vegas Film Critics Society Award for Best Actress'],\n", + " ['Ida Random', 'people.person.nationality', 'United States of America'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Directed by', 'rdf-schema#domain', 'Film'],\n", + " ['m.0y61751', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0zdsyts', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['Harvard College',\n", + " 'education.educational_institution.campuses',\n", + " 'Harvard College'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.05nwcn_'],\n", + " ['Wholphin: Issue 10',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Indie Dramas'],\n", + " [\"Paris, je t'aime\", 'film.film.produced_by', 'Emmanuel Benbihy'],\n", + " ['No Strings Attached', 'freebase.valuenotation.is_reviewed', 'Directed by'],\n", + " ['Marco Beltrami', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.07zq3__', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Location'],\n", + " ['Love and Other Impossible Pursuits', 'common.topic.notable_types', 'Film'],\n", + " ['m.0gx8_vd', 'freebase.valuenotation.is_reviewed', 'Type of union'],\n", + " ['Yvan Attal', 'people.person.religion', 'Judaism'],\n", + " ['Lukus Haas', 'base.popstra.celebrity.dated', 'm.065q1bw'],\n", + " ['m.0gx8_vd', 'freebase.valuenotation.is_reviewed', 'To'],\n", + " ['m.064swgn', 'base.popstra.vacation_choice.location', 'Morocco'],\n", + " ['Samuel \\\\\"Mouli\\\\\" Cohen', 'common.topic.notable_types', 'Person'],\n", + " ['2002 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0_h8yyt'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tj8r'],\n", + " ['Teen Choice Award for Choice Movie Actress',\n", + " 'award.award_category.nominees',\n", + " 'm.0gm46n_'],\n", + " ['Children', 'rdf-schema#domain', 'Person'],\n", + " ['No Strings Attached', 'film.film.other_crew', 'm.0g4ndyp'],\n", + " ['Hesher', 'film.film.release_date_s', 'm.0qsjvs9'],\n", + " ['Mike Nichols', 'people.person.nationality', 'United States of America'],\n", + " ['Grace Cahill', 'film.film_character.portrayed_in_films', 'm.040myw2'],\n", + " ['m.07l4_nl', 'film.film_cut.film', 'Love and Other Impossible Pursuits'],\n", + " ['Black Swan', 'award.award_nominated_work.award_nominations', 'm.0fpnkn8'],\n", + " ['m.0v1zs_9', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['m.0pc689n', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Shiri Appleby',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['MA$TADON', 'people.person.profession', 'Screenwriter'],\n", + " ['m.0y62dtl', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0v1zqgb', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['National Board of Review Award for Best Cast',\n", + " 'award.award_category.winners',\n", + " 'm.0gm475f'],\n", + " ['Film director', 'common.topic.subject_of', 'Eugenio Polgovsky'],\n", + " ['Israel', 'common.topic.notable_types', 'Country'],\n", + " ['m.0w82r7j',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Jane Got a Gun'],\n", + " ['Garden State',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Romantic Movies'],\n", + " ['Anna Boleyn', 'film.film_character.portrayed_in_films', 'm.02vdb41'],\n", + " ['Israelis', 'people.ethnicity.people', 'Reuven Rivlin'],\n", + " ['Peter Fruchtman', 'people.person.gender', 'Male'],\n", + " ['Jude Law', 'award.award_winner.awards_won', 'm.0gm475f'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0z9x3f_'],\n", + " ['A Powerful Noise Live', 'film.film.country', 'United States of America'],\n", + " ['m.012sr3qf', 'people.human_measurement.person', 'Natalie Portman'],\n", + " ['Hesher', 'film.film.starring', 'm.0vnwrdx'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0b6q7kd'],\n", + " ['No Strings Attached', 'film.film.release_date_s', 'm.0j584zj'],\n", + " ['m.0gm46p9',\n", + " 'award.award_nomination.award',\n", + " 'London Critics Circle Film Award for Actress of the Year'],\n", + " ['From', 'rdf-schema#domain', 'Marriage'],\n", + " ['Javier Bardem', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Dan Bucatinsky', 'common.topic.notable_types', 'TV Writer'],\n", + " ['Brothers', 'media_common.netflix_title.netflix_genres', 'Drama'],\n", + " ['The 77th Annual Academy Awards',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['Veganism', 'base.eating.diets.avoids_food_type', 'Animal product'],\n", + " ['No Strings Attached', 'film.film.runtime', 'm.0gcllx3'],\n", + " ['m.0gfsghz', 'common.webpage.topic', 'Hesher'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Drama'],\n", + " ['Free Zone', 'film.film.starring', 'm.0jygjj'],\n", + " ['Ehud Olmert', 'people.person.nationality', 'Israel'],\n", + " ['m.02vdb41', 'film.performance.film', 'The Other Boleyn Girl'],\n", + " ['Hayden Christensen', 'award.award_nominee.award_nominations', 'm.0z9wqpt'],\n", + " ['m.09zl530',\n", + " 'award.award_nomination.ceremony',\n", + " \"10th Critics' Choice Awards\"],\n", + " ['Nathan Bogle', 'base.popstra.celebrity.dated', 'm.065q80s'],\n", + " ['m.0w9tj4y',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Him/Herself'],\n", + " ['Film Producer', 'film.film_job.films_with_this_crew_job', 'm.09dw8m_'],\n", + " ['m.0zdv4fv', 'film.film_crew_gig.film', 'Jane Got a Gun'],\n", + " ['To', 'rdf-schema#range', 'Date/Time'],\n", + " ['Israelis', 'people.ethnicity.languages_spoken', 'Arabic Language'],\n", + " ['Bryce Dallas Howard', 'people.person.profession', 'Film Producer'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0gkm43g'],\n", + " ['Heat', 'film.film.genre', 'Drama'],\n", + " ['Shaul Mofaz', 'people.person.nationality', 'Israel'],\n", + " ['Mila Kunis', 'award.award_nominee.award_nominations', 'm.0fq7b0k'],\n", + " ['Raviv Ullman', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0w9tktt', 'tv.tv_guest_personal_appearance.person', 'Natalie Portman'],\n", + " ['m.0gm475f', 'award.award_honor.award_winner', 'Clive Owen'],\n", + " ['Hesher', 'film.film.estimated_budget', 'm.099fmzh'],\n", + " ['Star Wars Episode II: Attack of the Clones',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z9wm_1'],\n", + " ['m.0h9hbwk', 'film.film_cut.film', 'Hesher'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.012zftn3'],\n", + " ['Everyone Says I Love You', 'film.film.starring', 'm.0jxysn'],\n", + " ['m.0gkkdsl', 'film.performance.film', 'Hesher'],\n", + " ['Jonathan Glickman', 'people.person.profession', 'Film Producer'],\n", + " ['Chris Parnell', 'music.artist.track_contributions', 'm.0rqvq41'],\n", + " ['m.0w4f1cj',\n", + " 'award.award_honor.award',\n", + " 'Boston Society of Film Critics Award for Best Actress'],\n", + " ['m.0w9tk6b', 'tv.tv_guest_personal_appearance.episode', 'Natalie Portman'],\n", + " ['m.0h70f_l', 'film.film_crew_gig.film_crew_role', 'Film Producer'],\n", + " ['Arnon Milchan', 'people.person.profession', 'Actor'],\n", + " ['Avi Arad', 'people.person.gender', 'Male'],\n", + " ['m.0jxysn', 'film.performance.film', 'Everyone Says I Love You'],\n", + " ['m.0gx8_vd', 'freebase.valuenotation.is_reviewed', 'Spouse'],\n", + " ['Washington, D.C.',\n", + " 'film.film_location.featured_in_films',\n", + " 'Mars Attacks!'],\n", + " ['Mars Attacks!',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Free Zone', 'film.film.directed_by', 'Amos Gitai'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0cgn67d'],\n", + " ['m.0gm46q7',\n", + " 'award.award_nomination.award',\n", + " 'San Diego Film Critics Society Award for Best Supporting Actress'],\n", + " ['Film Producer', 'film.film_job.films_with_this_crew_job', 'm.0w2wj6w'],\n", + " ['Jeanine Lobell', 'people.person.gender', 'Female'],\n", + " ['Hesher', 'film.film.genre', 'Drama'],\n", + " ['Natalie Portman/Fall Out Boy',\n", + " 'base.saturdaynightlive.snl_episode.musical_guest',\n", + " 'Fall Out Boy'],\n", + " [\"Paris, je t'aime\", 'common.topic.notable_types', 'Film'],\n", + " ['m.0zdqwn0', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['Cassian Elwes', 'freebase.valuenotation.is_reviewed', 'Siblings'],\n", + " ['m.0g4xthd', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['Location of ceremony', 'rdf-schema#domain', 'Marriage'],\n", + " ['Jane Got a Gun', 'film.film.edited_by', 'Alan Cody'],\n", + " ['Thor', 'film.film.country', 'United States of America'],\n", + " ['m.0cp8_gl', 'film.performance.film', 'No Strings Attached'],\n", + " ['Knight of Cups', 'common.topic.notable_types', 'Film'],\n", + " ['Jewish people', 'people.ethnicity.included_in_group', 'Semitic people'],\n", + " ['New York, I Love You',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Indie Comedies'],\n", + " ['Black Swan', 'film.film.rating', 'R (USA)'],\n", + " ['Zach Braff', 'award.award_nominee.award_nominations', 'm.0b3w_yx'],\n", + " ['Natalie Portman - TIFF2010 01', 'common.image.size', 'm.029g9xn'],\n", + " ['Marc E. Platt',\n", + " 'film.producer.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Avi Arad', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['No Strings Attached', 'film.film.genre', 'Romantic comedy'],\n", + " ['Chaim Weizmann', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Leah Goldberg', 'people.person.ethnicity', 'Israelis'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0fq5c1f'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0gkm44d'],\n", + " ['Hesher', 'film.film.executive_produced_by', 'Wayne Chang'],\n", + " ['m.0k0x4n', 'film.performance.film', \"Mr. Magorium's Wonder Emporium\"],\n", + " ['Wholphin: Issue 10', 'film.film.personal_appearances', 'm.0cs9p0f'],\n", + " ['Lisa Kudrow', 'people.person.gender', 'Female'],\n", + " ['Chicago Film Critics Association Award for Best Actress',\n", + " 'award.award_category.winners',\n", + " 'm.0w4f18c'],\n", + " ['Hesher', 'film.film.genre', 'Family Drama'],\n", + " ['Cross Creek Pictures',\n", + " 'film.production_company.films',\n", + " 'Pride and Prejudice and Zombies'],\n", + " ['Guy Oseary', 'people.person.profession', 'Actor'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0t4tdr_'],\n", + " ['Oren Moverman', 'people.person.place_of_birth', 'Israel'],\n", + " ['Israelis', 'people.ethnicity.people', 'Ovadia Yosef'],\n", + " ['Jane Got a Gun', 'film.film.other_crew', 'm.0zdv4_n'],\n", + " ['Justine Baddeley',\n", + " 'film.film_casting_director.films_casting_directed',\n", + " 'Hesher'],\n", + " ['Dana E. Glauberman',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date of birth'],\n", + " ['m.0y60y38', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0w9thmr', 'tv.tv_guest_personal_appearance.episode', 'Show #1393'],\n", + " ['Film Producer',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0106b7nl'],\n", + " ['Black Swan', 'award.award_nominated_work.award_nominations', 'm.0fpy598'],\n", + " ['Actor', 'common.topic.subjects', 'Chris Free'],\n", + " ['Hebrew University of Jerusalem',\n", + " 'education.educational_institution_campus.educational_institution',\n", + " 'Hebrew University of Jerusalem'],\n", + " ['m.0w4dzvx',\n", + " 'award.award_honor.award',\n", + " 'Scream Awards Best Fantasy Actress'],\n", + " ['Gene Simmons', 'people.person.ethnicity', 'Jewish people'],\n", + " ['m.0z9qljp', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Clive Owen', 'award.award_nominee.award_nominations', 'm.09k3pxr'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0g4nfqh'],\n", + " ['Star Wars Episode II: Attack of the Clones',\n", + " 'film.film.language',\n", + " 'English Language'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0cp8_gl'],\n", + " ['Black Swan: Metamorphosis', 'film.film.language', 'English Language'],\n", + " ['Black Swan', 'film.film.country', 'United States of America'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60zcc'],\n", + " ['m.0ywnbsz', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Place of birth', 'type.property.schema', 'Person'],\n", + " ['Mike Nichols', 'people.person.profession', 'Film director'],\n", + " ['New York, I Love You', 'film.film.directed_by', 'Joshua Marston'],\n", + " ['Gael García Bernal', 'people.person.profession', 'Film director'],\n", + " ['Film director', 'common.topic.subjects', 'Eugenio Polgovsky'],\n", + " ['Marriage', 'freebase.equivalent_topic.equivalent_type', 'Marriage'],\n", + " ['Israelis', 'people.ethnicity.people', 'Chaim Herzog'],\n", + " ['m.0fq5c1f',\n", + " 'award.award_nomination.award',\n", + " 'Golden Globe Award for Best Actress – Motion Picture – Drama'],\n", + " ['m.0gm46n_', 'award.award_nomination.ceremony', '2005 Teen Choice Awards'],\n", + " ['Avi Lerner', 'people.person.nationality', 'Israel'],\n", + " ['Natalie Portman', 'people.person.places_lived', 'm.0cg_cbs'],\n", + " ['Yvan Attal', 'film.director.film', 'New York, I Love You'],\n", + " ['m.0b3w_yx', 'award.award_nomination.award_nominee', 'Zach Braff'],\n", + " ['Marco Beltrami', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0w9tjj2', 'tv.tv_guest_personal_appearance.episode', 'Natalie Portman'],\n", + " ['Ralph Bakshi', 'people.person.nationality', 'Israel'],\n", + " ['National Board of Review Award for Best Cast',\n", + " 'award.award_category.nominees',\n", + " 'm.0gm46pt'],\n", + " ['Ephraim Katzir', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Gael García Bernal', 'people.person.profession', 'Film Producer'],\n", + " ['Wholphin: Issue 10', 'film.film.genre', 'Indie film'],\n", + " ['m.0cl665w', 'common.webpage.topic', 'Eve'],\n", + " ['New York, I Love You', 'film.film.genre', 'Drama'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0p433rm'],\n", + " ['Natalie Portman', 'award.award_winner.awards_won', 'm.0_h8yyt'],\n", + " ['m.0g5jf42', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['m.0z87rqd', 'award.award_honor.honored_for', 'No Strings Attached'],\n", + " ['Gal Gadot', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Ayelet Zurer', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Jane Got a Gun', 'film.film.written_by', 'Brian Duffield'],\n", + " ['Arcadi Gaydamak', 'people.person.ethnicity', 'Israelis'],\n", + " ['Nathan Rothschild, 1st Baron Rothschild',\n", + " 'base.popstra.celebrity.dated',\n", + " 'm.065pvp_'],\n", + " [\"Mr. Magorium's Wonder Emporium\", 'film.film.starring', 'm.0k0x4n'],\n", + " ['Veganism', 'food.diet.recipes', 'Pasta with Simple Tomato Sauce'],\n", + " ['Chaim Weizmann', 'people.person.nationality', 'United Kingdom'],\n", + " ['Remi Adefarasin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.07pmhpl', 'freebase.valuenotation.has_value', 'To'],\n", + " ['m.0cs2bt7', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Ayelet Zurer', 'people.person.gender', 'Female'],\n", + " ['Celebrity', 'type.type.domain', 'Celebrities'],\n", + " ['Eugenio Polgovsky', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0w9tjwv',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Natalie Portman/Gary Marshall'],\n", + " ['m.0g4xwxy',\n", + " 'film.film_crew_gig.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['m.0cg_c8x', 'education.education.student', 'Natalie Portman'],\n", + " ['Veganism', 'common.topic.subject_of', 'Mike Arnesen'],\n", + " ['m.0qsj_zz', 'film.film_regional_release_date.film', 'Hesher'],\n", + " ['Natalie Portman', 'base.popstra.celebrity.canoodled', 'm.0652t7d'],\n", + " ['m.0g5lh1f', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Mila Kunis', 'award.award_nominee.award_nominations', 'm.0z87_sd'],\n", + " ['Remi Adefarasin',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0gm46m_'],\n", + " ['No Strings Attached', 'film.film.music', 'John Debney'],\n", + " ['Jane Got a Gun', 'film.film.produced_by', 'Scott LaStaiti'],\n", + " ['Lisa Gerrard', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Anywhere but Here', 'film.film.country', 'United States of America'],\n", + " ['m.0gx90td', 'film.performance.film', 'True'],\n", + " ['m.0qsk1lm', 'film.film_regional_release_date.film', 'Hesher'],\n", + " ['My Blueberry Nights',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Indie Dramas'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tk0s'],\n", + " ['m.0gx8_vd', 'people.marriage.spouse', 'Natalie Portman'],\n", + " ['m.0b3w_yx',\n", + " 'award.award_nomination.award',\n", + " 'MTV Movie Award for Best Kiss'],\n", + " ['m.065qbcb', 'base.popstra.dated.participant', 'Andy Samberg'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0v1_4dn'],\n", + " ['Gal Gadot', 'people.person.ethnicity', 'Israelis'],\n", + " ['Hebrew Language', 'language.human_language.countries_spoken_in', 'Israel'],\n", + " ['Moshe Dayan', 'people.person.children', 'Yael Dayan'],\n", + " ['m.0nfwhqy', 'award.award_honor.award', 'SFX Award for Best Actress'],\n", + " ['Peter Fruchtman', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Academy Award for Best Actress',\n", + " 'award.award_category.winners',\n", + " 'm.0gdjwkg'],\n", + " ['m.010wr3t7', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['m.0w4f0w8',\n", + " 'award.award_honor.ceremony',\n", + " 'Florida Film Critics Circle Awards 2010'],\n", + " ['m.09k3pxr', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", + " ['Thor: The Dark World',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Directed by'],\n", + " ['Devendra Banhart', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0b3t_f0'],\n", + " ['Julia Roberts', 'people.person.languages', 'English Language'],\n", + " ['m.0gx8_vd', 'people.marriage.type_of_union', 'Domestic partnership'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.film_production_design_by',\n", + " 'Michael Shaw'],\n", + " ['m.0y8br6l', 'freebase.valuenotation.has_no_value', 'To'],\n", + " ['m.0j_xb7', 'film.performance.actor', 'Natalie Portman'],\n", + " ['Beautiful Girls', 'media_common.netflix_title.netflix_genres', 'Comedies'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9thy2'],\n", + " ['m.0fq7b0k', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Elizabeth Meriwether',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Dan Bucatinsky', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0131yzfn', 'film.performance.film', 'Pride and Prejudice and Zombies'],\n", + " ['Film Producer', 'common.topic.subject_of', 'Lew M. Parry'],\n", + " ['Jonathan Safran Foer',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['m.07pmhpl', 'business.employment_tenure.person', 'Natalie Portman'],\n", + " ['m.0w4dzf8', 'award.award_honor.honored_for', 'Black Swan'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0pllr38'],\n", + " ['Film Producer',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0_z5h42'],\n", + " ['Initial release date', 'rdf-schema#range', 'Date/Time'],\n", + " ['m.0gkm42b', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Hesher', 'film.film.starring', 'm.0gkkdpm'],\n", + " ['Eyal Podell', 'people.person.gender', 'Male'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.other_crew', 'm.0g4xwxy'],\n", + " ['Israel', 'location.country.languages_spoken', 'Hebrew Language'],\n", + " ['Guy Oseary', 'people.person.gender', 'Male'],\n", + " ['Evey Hammond', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['No Strings Attached', 'common.topic.notable_for', 'g.1254xcq4m'],\n", + " ['Celebrity', 'freebase.type_profile.published', 'Published'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.directed_by', 'Burr Steers'],\n", + " ['m.0gx8_s8', 'people.place_lived.location', 'Washington, D.C.'],\n", + " ['Ayelet Zurer', 'people.person.ethnicity', 'Israelis'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60y38'],\n", + " ['m.0pdhvh0',\n", + " 'award.award_nomination.ceremony',\n", + " '23rd Golden Raspberry Awards'],\n", + " ['m.0g8mcmc',\n", + " 'award.award_nomination.award',\n", + " 'Academy Award for Best Actress'],\n", + " ['m.0g4nfqw', 'film.performance.film', 'No Strings Attached'],\n", + " ['Black Swan', 'film.film.starring', 'm.0cccx3m'],\n", + " ['Spouse', 'rdf-schema#range', 'Person'],\n", + " ['m.0zdqslk', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['Natalie Portman',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0w0wbjd'],\n", + " ['Eve', 'film.film.edited_by', 'Tricia Cooke'],\n", + " ['Burr Steers', 'film.director.film', 'Pride and Prejudice and Zombies'],\n", + " ['Natalie Portman', 'film.actor.film', 'm.0vpf5kj'],\n", + " ['Jane Got a Gun', 'film.film.release_date_s', 'm.0zdpyjl'],\n", + " ['Jude Law', 'people.person.languages', 'English Language'],\n", + " ['m.0y60zkt', 'film.performance.film', 'No Strings Attached'],\n", + " ['m.0zdrw0v', 'film.performance.film', 'Jane Got a Gun'],\n", + " ['Israelis', 'common.topic.notable_types', 'Ethnicity'],\n", + " ['m.0v201x0', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['Léon: The Professional',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Featured Song'],\n", + " ['Remi Adefarasin', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Pride and Prejudice and Zombies',\n", + " 'film.film.production_companies',\n", + " 'Cross Creek Pictures'],\n", + " ['Film Producer',\n", + " 'freebase.equivalent_topic.equivalent_type',\n", + " 'Film producer'],\n", + " ['No Strings Attached', 'film.film.executive_produced_by', 'Gary Barber'],\n", + " ['Black Swan', 'award.award_nominated_work.award_nominations', 'm.0g8mcmc'],\n", + " ['Yitzhak Rabin', 'people.person.place_of_birth', 'Jerusalem'],\n", + " ['m.0w4f166', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['Hesher', 'film.film.production_companies', 'The Last Picture Company'],\n", + " ['Hesher', 'film.film.genre', 'Childhood Drama'],\n", + " ['Hebrew Language', 'language.human_language.main_country', 'Israel'],\n", + " ['Closer', 'film.film.starring', 'm.0jwjgq'],\n", + " ['Jewish people', 'people.ethnicity.languages_spoken', 'Hebrew Language'],\n", + " ['Emmanuel Benbihy', 'people.person.profession', 'Film Producer'],\n", + " ['m.0j4jpz0', 'freebase.valuenotation.has_value', 'From'],\n", + " ['Israelis', 'people.ethnicity.languages_spoken', 'Russian Language'],\n", + " ['Hayden Christensen', 'people.person.profession', 'Model'],\n", + " ['Benjamin Millepied', 'people.person.children', 'Aleph Portman-Millepied'],\n", + " ['m.0_h8yyt',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Movie Actress: Action'],\n", + " ['m.0hj8mwx', 'freebase.valuenotation.is_reviewed', 'Award'],\n", + " ['Ran Danker', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Mars Attacks!', 'film.film.genre', 'Thriller'],\n", + " ['Raviv Ullman',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60_p3'],\n", + " ['Leah Goldberg', 'people.deceased_person.place_of_death', 'Jerusalem'],\n", + " ['m.0vpf5kj', 'film.performance.character', 'Alicia'],\n", + " ['m.0pc689n', 'award.award_nomination.award_nominee', 'Mila Kunis'],\n", + " ['Garden State', 'film.film.rating', 'R (USA)'],\n", + " ['Zoolander', 'film.film.production_companies', 'Paramount Pictures'],\n", + " ['Lisa Kudrow', 'people.person.profession', 'Actor'],\n", + " ['Adam Levine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0y611qv', 'film.performance.film', 'No Strings Attached'],\n", + " ['Natalie Portman', 'people.measured_person.sizes', 'm.012sr409'],\n", + " ['Avi Lerner', 'people.person.profession', 'Film Producer'],\n", + " ['m.0hzv4rl',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Avi Arad', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['62nd Golden Globe Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.07_2z79'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0g4nfq7'],\n", + " ['17th Screen Actors Guild Awards',\n", + " 'award.award_ceremony.nominees',\n", + " 'm.0fq7b0k'],\n", + " ['Roman Zaretsky', 'people.person.ethnicity', 'Israelis'],\n", + " ['Jude Law', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Initial release date', 'type.property.expected_type', 'Date/Time'],\n", + " ['Yitzhak Ben-Zvi', 'people.person.nationality', 'Israel'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0v201x0'],\n", + " ['TV Writer', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Ehud Olmert', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.country', 'United Kingdom'],\n", + " ['m.0vnxj9r', 'film.performance.film', 'Hesher'],\n", + " [\"Paris, je t'aime\", 'film.film.genre', 'Romance Film'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.genre', 'Thriller'],\n", + " ['m.0h12kcc', 'film.personal_film_appearance.person', 'Natalie Portman'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.012yc5s8'],\n", + " ['Pride and Prejudice and Zombies',\n", + " 'film.film.production_companies',\n", + " 'Dark Films'],\n", + " ['m.0jtns5', 'film.performance.character', 'Ann August'],\n", + " ['Start Date', 'type.property.expected_type', 'Date/Time'],\n", + " ['Celebrity', 'type.type.properties', 'Celebrity friends'],\n", + " ['Date of birth', 'rdf-schema#range', 'Date/Time'],\n", + " ['m.0t4tcxl', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Jeanine Lobell', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['m.0k3qy8',\n", + " 'film.performance.film',\n", + " 'Star Wars Episode I: The Phantom Menace'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.film_casting_director',\n", + " 'Bernard Telsey'],\n", + " ['Ivan Reitman', 'people.person.gender', 'Male'],\n", + " ['Natalie Portman', 'common.topic.image', 'Natalie Portman'],\n", + " ['Natalie Portman, Mark Consuelos, Los Lobos',\n", + " 'common.topic.notable_types',\n", + " 'TV Episode'],\n", + " [\"Paris, je t'aime\", 'media_common.netflix_title.netflix_genres', 'French'],\n", + " ['Parents', 'rdf-schema#domain', 'Person'],\n", + " ['Israelis', 'common.topic.image', 'Dana International'],\n", + " ['Laura Dandridge', 'film.film_character.portrayed_in_films', 'm.0jxysn'],\n", + " ['Abby and Friends: P Is for Princess',\n", + " 'common.topic.notable_types',\n", + " 'TV Episode'],\n", + " ['m.0v1_qft', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['m.0p75gts',\n", + " 'award.award_honor.ceremony',\n", + " '17th Screen Actors Guild Awards'],\n", + " ['Initial release date', 'type.property.schema', 'Film'],\n", + " ['Isaac Mizrahi',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Free Zone', 'film.film.genre', 'Drama'],\n", + " ['m.090dzg7', 'award.award_nomination.ceremony', '57th Golden Globe Awards'],\n", + " ['Ethnicity', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.genre', 'Romance Film'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0gm46p9'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60z23'],\n", + " ['m.0y615_l', 'film.performance.film', 'No Strings Attached'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0cp8_db'],\n", + " ['Natalie Portman', 'people.person.education', 'm.02wmw6l'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Washington, D.C.'],\n", + " ['Raviv Ullman', 'people.person.ethnicity', 'Israelis'],\n", + " ['Reuven Rivlin', 'people.person.gender', 'Male'],\n", + " ['Dallas–Fort Worth Film Critics Association Award for Best Actress',\n", + " 'award.award_category.winners',\n", + " 'm.0w4f166'],\n", + " ['Christophe Beck', 'people.person.gender', 'Male'],\n", + " ['m.0qsk0ty', 'film.film_regional_release_date.film', 'Hesher'],\n", + " ['m.0w9tj1c',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Grass is Always Greener'],\n", + " ['Christophe Beck', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tjwv'],\n", + " ['Jane Got a Gun', 'film.film.other_crew', 'm.0zdv43b'],\n", + " ['Your Highness', 'common.topic.notable_types', 'Film'],\n", + " ['m.0jynxn', 'film.performance.character', 'Sam'],\n", + " ['Natalie Portman', 'base.popstra.celebrity.dated', 'm.065qbcb'],\n", + " ['Celebrity', 'freebase.equivalent_topic.equivalent_type', 'Celebrity'],\n", + " ['2011 MTV Movie Awards', 'award.award_ceremony.nominees', 'm.0pc6s99'],\n", + " ['No Strings Attached',\n", + " 'film.film.production_companies',\n", + " 'Cold Spring Pictures'],\n", + " ['Celebrity', 'type.type.expected_by', 'Partners'],\n", + " ['Jake Gyllenhaal', 'people.person.ethnicity', 'Jewish people'],\n", + " ['Shlomo Ben-Ami', 'people.person.ethnicity', 'Jewish people'],\n", + " ['m.0qsjsh5',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'United States of America'],\n", + " ['Gael García Bernal',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date of birth'],\n", + " ['Closer', 'award.award_nominated_work.award_nominations', 'm.0gm46m_'],\n", + " ['m.09k3pxr',\n", + " 'award.award_nomination.award',\n", + " \"Critics' Choice Movie Award for Best Acting Ensemble\"],\n", + " ['Eve', 'film.film.film_art_direction_by', 'Jennifer Dehghan'],\n", + " ['m.0gkfdxg', 'common.webpage.topic', 'Hesher'],\n", + " ['Garden State', 'freebase.valuenotation.is_reviewed', 'Directed by'],\n", + " ['Ida Random', 'people.person.gender', 'Female'],\n", + " ['Jane Got a Gun', 'film.film.starring', 'm.0nfnhrj'],\n", + " ['Garden State', 'film.film.genre', 'Drama'],\n", + " ['Jake Gyllenhaal', 'people.person.profession', 'Actor'],\n", + " ['Vincent Cassel', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['No Strings Attached',\n", + " 'film.film.soundtrack',\n", + " 'No Strings Attached: Music from the Motion Picture'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0cp8_hb'],\n", + " ['Zoolander', 'media_common.netflix_title.netflix_genres', 'Comedy'],\n", + " ['m.0z87tfb', 'freebase.valuenotation.has_no_value', 'Award Nominee'],\n", + " ['Barbara Hershey', 'people.person.profession', 'Actor'],\n", + " ['No Strings Attached',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0pc6s99'],\n", + " ['Shelley Stevens', 'common.topic.notable_types', 'Person'],\n", + " ['Ivan Reitman', 'people.person.profession', 'Actor'],\n", + " ['Isaac Mizrahi', 'people.person.nationality', 'United States of America'],\n", + " ['No Strings Attached',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Comedy'],\n", + " ['Natalie Portman', 'film.actor.film', 'm.0jv4ds'],\n", + " ['Hesher', 'film.film.runtime', 'm.0gfrlr2'],\n", + " ['Francine', 'film.film_character.portrayed_in_films', 'm.0cs2bt7'],\n", + " ['New York, I Love You', 'film.film.genre', 'Romance Film'],\n", + " ['m.059phw1', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Film', 'type.type.properties', 'Featured Song'],\n", + " ['Dana E. Glauberman', 'people.person.gender', 'Female'],\n", + " ['Hayden Christensen', 'people.person.profession', 'Film Producer'],\n", + " ['m.02vb_fz', 'film.performance.film', 'The Darjeeling Limited'],\n", + " ['Yitzhak Ben-Zvi', 'people.person.ethnicity', 'Israelis'],\n", + " ['Natalie Portman', 'film.writer.film', 'New York, I Love You'],\n", + " ['Zac Posen',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'Zac Posen'],\n", + " ['Gender', 'rdf-schema#range', 'Gender'],\n", + " ['Shiri Appleby', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Devendra Banhart', 'people.person.profession', 'Actor'],\n", + " ['Hesher', 'common.topic.article', 'm.0b6d3ks'],\n", + " ['m.065q1bw', 'base.popstra.dated.participant', 'Lukus Haas'],\n", + " ['m.0g4xqxm',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'United States of America'],\n", + " ['Yitzhak Ben-Zvi', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Natalie Portman', 'film.writer.film', 'Eve'],\n", + " ['Lisa Gerrard',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.05nwcp4'],\n", + " ['Winona Ryder', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Place of birth', 'rdf-schema#range', 'Location'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0g5lh1f'],\n", + " ['m.0gx8_vd', 'people.marriage.spouse', 'Benjamin Millepied'],\n", + " ['Andy Samberg', 'people.person.profession', 'Film Producer'],\n", + " ['Closer', 'award.award_winning_work.awards_won', 'm.0gm4770'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0cp8_8z'],\n", + " ['Show #0914', 'common.topic.notable_types', 'TV Episode'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60wk6'],\n", + " ['Roger Birnbaum',\n", + " 'film.producer.films_executive_produced',\n", + " 'No Strings Attached'],\n", + " ['Teen Choice Award for Choice Movie Love Scene',\n", + " 'award.award_category.nominees',\n", + " 'm.0z9s_qp'],\n", + " ['Bryce Dallas Howard', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Natalie Portman', 'tv.tv_actor.guest_roles', 'm.0w9j5t_'],\n", + " ['Dan Bucatinsky',\n", + " 'film.producer.films_executive_produced',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['m.0gkm438', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['No Strings Attached', 'film.film.produced_by', 'Joe Medjuck'],\n", + " ['Benjamin Millepied', 'people.person.profession', 'Actor'],\n", + " ['Shimon Peres', 'people.person.religion', 'Judaism'],\n", + " ['Ralph Bakshi', 'people.person.ethnicity', 'Israeli American'],\n", + " ['m.0y6c70k', 'film.performance.film', 'No Strings Attached'],\n", + " ['2005 MTV Movie Awards', 'award.award_ceremony.nominees', 'm.0b3t_f0'],\n", + " [\"Milos Forman: What doesn't kill you\",\n", + " 'common.topic.notable_types',\n", + " 'Film'],\n", + " ['Natalie Portman',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'Natalie Portman'],\n", + " ['Chaim Herzog', 'people.person.ethnicity', 'Israelis'],\n", + " ['Hesher',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Independent Movies'],\n", + " ['m.0k3r0b', 'film.performance.character', 'Padmé Amidala'],\n", + " ['m.0vnw_y_', 'film.performance.film', 'Hesher'],\n", + " ['Reuven Rivlin', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Yiddish Language',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'Israel'],\n", + " ['Wholphin: Issue 10', 'film.film.directed_by', 'Natalie Portman'],\n", + " ['Amos Oz', 'people.person.gender', 'Male'],\n", + " ['Natalie Portman', 'tv.tv_program_guest.appeared_on', 'm.0w9tjm4'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.starring', 'm.0v1_bw5'],\n", + " ['Arthur Edelstein',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['m.0t4td2m', 'film.film_crew_gig.film', 'Hesher'],\n", + " [\"Goya's Ghosts\", 'film.film.starring', 'm.0vpf5kj'],\n", + " ['m.0nfnhzn', 'film.performance.film', 'Knight of Cups'],\n", + " ['Siblings', 'type.property.schema', 'Person'],\n", + " ['Kim Davis', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['Ariel Sharon', 'people.person.ethnicity', 'Jewish people'],\n", + " ['The Other Boleyn Girl', 'film.film.country', 'United States of America'],\n", + " ['Gal Gadot', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0w2wj7h', 'film.film_crew_gig.film_crew_role', 'Film Producer'],\n", + " ['Hesher', 'film.film.edited_by', 'Spencer Susser'],\n", + " ['m.0z9wqpt', 'award.award_nomination.award_nominee', 'Hayden Christensen'],\n", + " ['Ivan Reitman', 'people.person.profession', 'Film Producer'],\n", + " ['Mars Attacks!', 'freebase.valuenotation.is_reviewed', 'Directed by'],\n", + " ['Guy Oseary', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y60xxj'],\n", + " ['Love and Other Impossible Pursuits', 'film.film.other_crew', 'm.0g4xwvp'],\n", + " ['m.059phw1',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Star Wars Episode II: Attack of the Clones'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Robert Aumann'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0gkm438'],\n", + " ['Hesher', 'film.film.produced_by', 'Lucy Cooper'],\n", + " ['Celebrity', 'freebase.type_profile.kind', 'Description'],\n", + " ['m.0y62hn2', 'film.performance.film', 'No Strings Attached'],\n", + " ['Israel Horovitz', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Dana E. Glauberman',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Chicago Film Critics Association Awards 2010',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0w4f18c'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y618d8'],\n", + " ['Barbara Hershey', 'people.person.ethnicity', 'Jewish people'],\n", + " ['m.0gx8_vd', 'freebase.valuenotation.has_no_value', 'Location of ceremony'],\n", + " ['Person', 'type.type.properties', 'Children'],\n", + " ['Natalie Portman', 'award.award_nominee.award_nominations', 'm.0pc689n'],\n", + " ['Andy Samberg', 'people.person.profession', 'Actor'],\n", + " ['The Darjeeling Limited',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Indie film'],\n", + " ['m.0z87_sd',\n", + " 'award.award_nomination.award',\n", + " 'Teen Choice Award for Choice Movie: Liplock'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0cp8_cz'],\n", + " ['Tricia Cooke', 'people.person.place_of_birth', 'United States of America'],\n", + " ['Israelis', 'people.ethnicity.people', 'Tzipi Livni'],\n", + " ['Celebrity', 'type.type.expected_by', 'Celebrity represented'],\n", + " ['Show #2882', 'common.topic.notable_types', 'TV Episode'],\n", + " ['m.0h5ntgs', 'film.performance.film', 'No Strings Attached'],\n", + " ['Clive Owen', 'people.person.profession', 'Actor'],\n", + " ['Jane Got a Gun', 'film.film.genre', 'Western'],\n", + " ['Kansas City Film Critics Circle Awards 2010',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0w4f0kn'],\n", + " ['Natalie Portman',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0cs1dh5'],\n", + " ['Natalie Portman', 'base.popstra.celebrity.dated', 'm.065q80s'],\n", + " ['Pride and Prejudice and Zombies', 'film.film.starring', 'm.012zfqf3'],\n", + " ['Natalie Portman', 'film.actor.film', 'm.0cp7w2m'],\n", + " ['Natalie Portman',\n", + " 'award.ranked_item.appears_in_ranked_lists',\n", + " 'm.07y34zd'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0gkm446'],\n", + " ['m.0g4xwv3',\n", + " 'film.film_crew_gig.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Carol Cuddy', 'people.person.gender', 'Female'],\n", + " ['Romance Film', 'media_common.media_genre.child_genres', 'Romantic comedy'],\n", + " ['Veganism', 'book.book_subject.works', 'The Conscious Cook'],\n", + " ['Ayelet Zurer', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0t4tcwg'],\n", + " ['m.0g4xww7',\n", + " 'film.film_crew_gig.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Jonathan Safran Foer', 'people.person.gender', 'Male'],\n", + " ['m.0fpnkn8', 'award.award_nomination.ceremony', '15th Satellite Awards'],\n", + " ['Weightless', 'film.film.starring', 'm.0nh4bk_'],\n", + " ['Ralph Bakshi', 'people.person.nationality', 'United States of America'],\n", + " ['m.0_h8yyt', 'award.award_honor.award_winner', 'Natalie Portman'],\n", + " ['m.012sr3yr', 'people.human_measurement.person', 'Natalie Portman'],\n", + " ['MA$TADON', 'common.topic.subject_of', 'Artist'],\n", + " ['m.0w9thjt',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Natalie Portman, Mark Consuelos, Los Lobos'],\n", + " ['Danielle Berman', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['No Strings Attached', 'film.film.other_crew', 'm.0g4ndyy'],\n", + " ['Hesher', 'film.film.other_crew', 'm.0t4td5f'],\n", + " ['No Strings Attached', 'film.film.country', 'United States of America'],\n", + " ['Black Swan', 'award.award_nominated_work.award_nominations', 'm.0z87s8m'],\n", + " ['m.0qsjltl',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'United States of America'],\n", + " ['Amos Oz', 'people.person.place_of_birth', 'Jerusalem'],\n", + " ['m.0g4xwwm',\n", + " 'film.film_crew_gig.film',\n", + " 'Love and Other Impossible Pursuits'],\n", + " ['Israelis', 'people.ethnicity.languages_spoken', 'Amharic Language'],\n", + " ['Film Producer', 'common.topic.subject_of', 'Eugenio Polgovsky'],\n", + " ['m.0pc6775', 'award.award_nomination.award_nominee', 'Natalie Portman'],\n", + " ['Israeli American', 'people.ethnicity.people', 'Oded Fehr'],\n", + " ['Morocco', 'common.topic.notable_types', 'Country'],\n", + " ['m.0vpf5kj', 'film.performance.film', \"Goya's Ghosts\"],\n", + " ['m.0nh4bnk', 'film.performance.actor', 'Natalie Portman'],\n", + " ['m.0n0l04n', 'freebase.valuenotation.has_value', 'End Date'],\n", + " ['Israel Horovitz', 'film.writer.film', 'New York, I Love You'],\n", + " ['Hesher', 'common.topic.webpage', 'm.0gkfdxs'],\n", + " ['Big Sur', 'base.biblioness.bibs_topic.subsumes', 'Big Sur'],\n", + " ['No Strings Attached', 'film.film.story_by', 'Michael Samonek'],\n", + " ['The Darjeeling Limited', 'film.film.genre', 'Drama'],\n", + " ['m.0v1_jc3', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['m.0qsk1s3', 'film.film_regional_release_date.film', 'Hesher'],\n", + " ['Place of birth', 'rdf-schema#domain', 'Person'],\n", + " ['Austin Film Critics Association Awards 2010',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0w4f1jr'],\n", + " ['Yonatan Netanyahu', 'people.person.gender', 'Male'],\n", + " ['m.0gcy8zl', 'film.performance.film', 'Hesher'],\n", + " ['Natalie Portman/Fall Out Boy',\n", + " 'base.saturdaynightlive.snl_episode.musical_guest_performance',\n", + " 'm.07ygf72'],\n", + " ['V for Vendetta', 'film.film.country', 'United States of America'],\n", + " ['MakingOf.com', 'business.employer.employees', 'm.07pmhpl'],\n", + " ['Show #1223', 'common.topic.notable_types', 'TV Episode'],\n", + " ['m.0t4tcyp', 'film.film_crew_gig.film', 'Hesher'],\n", + " ['Jane Got a Gun', 'common.topic.article', 'm.0swldz_'],\n", + " ['m.0gbm5y_',\n", + " 'award.award_honor.award',\n", + " 'BAFTA Award for Best Actress in a Leading Role'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Dramas Based on the Book'],\n", + " ['Ivan Reitman', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Knight of Cups', 'film.film.country', 'United States of America'],\n", + " ['m.0g4xtjt', 'film.performance.film', 'Love and Other Impossible Pursuits'],\n", + " ['No Strings Attached', 'film.film.story_by', 'Elizabeth Meriwether'],\n", + " ['Hesher',\n", + " 'film.film.film_festivals',\n", + " '2011 South by Southwest Film Festival'],\n", + " ['Eve', 'film.film.genre', 'Romantic comedy'],\n", + " ['m.05cgmx_', 'film.performance.film', 'Eve'],\n", + " ['Julia Roberts', 'base.popstra.celebrity.friendship', 'm.065pj6n'],\n", + " ['Ethnicity', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Lisa Kudrow', 'people.person.nationality', 'United States of America'],\n", + " ['m.05c79pt',\n", + " 'award.award_nomination.award',\n", + " 'Academy Award for Best Actress in a Supporting Role'],\n", + " ['The Other Boleyn Girl',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Dramas based on a book'],\n", + " ['No Strings Attached', 'film.film.starring', 'm.0y62fy6'],\n", + " ['Rahm Emanuel', 'people.person.ethnicity', 'Israeli American'],\n", + " ['Love and Other Impossible Pursuits',\n", + " 'film.film.executive_produced_by',\n", + " 'Dan Bucatinsky'],\n", + " ['Jane Got a Gun', 'film.film.executive_produced_by', 'Paris Latsis'],\n", + " ...],\n", + " [['Acklins',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Geographical Feature',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Location'],\n", + " ['Hurricane Edith',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1933 Treasure Coast hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4bpw0'],\n", + " ['m.04kc4ps', 'organization.organization_membership.member', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3s6q7'],\n", + " ['Politician', 'freebase.type_profile.equivalent_topic', 'Politician'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_mkq0'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.12460kjms'],\n", + " ['Tropical Storm Leslie',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Hurricane Flora',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'base.aareas.schema.administrative_area.pertinent_type',\n", + " 'District of the Bahamas'],\n", + " ['Hope Town',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Casino Royale', 'film.film.language', 'English Language'],\n", + " ['m.0khl_62', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas Hotel, Catering and Allied Workers Union',\n", + " 'organization.organization.geographic_scope',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Gregory Taylor'],\n", + " ['m.0khl4t6', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc3fsdd'],\n", + " ['m.064hd12', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Long Island', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['m.064ggl7', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4_k33'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_rq6l'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc3lmdf'],\n", + " ['Lucayan Archipelago',\n", + " 'geography.island_group.islands_in_group',\n", + " 'Paradise Island'],\n", + " ['Eastern Time Zone', 'common.topic.notable_for', 'g.125bn2wcf'],\n", + " ['Paradise Island',\n", + " 'location.location.contains',\n", + " 'Atlantis Royal Tower East'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4cg2_'],\n", + " ['North Abaco',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Airport', 'freebase.type_hints.included_types', 'Location'],\n", + " [\"Governor's Harbour Airport\", 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc41kjl'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc3fspl'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3sjfk'],\n", + " ['Hurricane Dennis',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'West Grand Bahama'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4f_1p'],\n", + " ['English Language',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'Bahamas'],\n", + " ['Geographical Feature', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Queen Victoria Platt', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Geographical Feature', 'common.topic.article', 'm.02hzhy1'],\n", + " ['Location', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245z78xk'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0j5n49x'],\n", + " ['Anya Watkins', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6fsy0'],\n", + " ['Marsh Harbour',\n", + " 'location.location.nearby_airports',\n", + " 'Marsh Harbour Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4wt5q'],\n", + " ['Green Turtle Cay District',\n", + " 'location.administrative_division.country',\n", + " 'Bahamas'],\n", + " ['Marvin Rolle', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4xvq8'],\n", + " ['Bahamas', 'location.location.contains', 'Acklins Island'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.11b71l32_5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4pgf5'],\n", + " ['Arthur Loves Plastic', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3hg97'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Bahamas Taxi Cab Union'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_qtbv'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Harbour Island'],\n", + " ['Andros Town International Airport',\n", + " 'aviation.airport.serves',\n", + " 'Andros Town'],\n", + " ['Jermaine Adderley', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3dz8d'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc4yk_f'],\n", + " ['Bahamas', 'common.topic.article', 'm.01615'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3k0rf'],\n", + " ['Hurricane Paloma',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc44p1_'],\n", + " ['Bahamas', 'location.location.contains', 'Mayaguana'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3m_r1'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc5052b'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc3pmqg'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124603c_3'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064sml9'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblm8'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'Bahamas Ministry of Foreign Affairs'],\n", + " ['Bahamas', 'location.location.contains', 'Bahama Banks'],\n", + " ['Grand Bahama International Airport',\n", + " 'location.location.geolocation',\n", + " 'm.02_jb0q'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4s82c'],\n", + " ['m.040tqy3',\n", + " 'government.political_party_tenure.politician',\n", + " 'Hubert Ingraham'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4twx7'],\n", + " ['Central Andros',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.country.administrative_divisions',\n", + " 'Cat Island, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4k6fj'],\n", + " ['The Assemblies of God Bible College, Nassau, NP, Bahamas',\n", + " 'education.educational_institution_campus.educational_institution',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['Alvin Smith', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1hhc38v7l'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc3jqfl'],\n", + " ['Eleuthera', 'common.topic.notable_types', 'Geographical Feature'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm7ss'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4p1t9'],\n", + " ['Cat Island, Bahamas', 'location.location.contains', 'Mount Alvernia'],\n", + " ['m.0jqg_bs', 'symbols.flag_use.flag_user', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc431x7'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'g.1245_rxx1'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlz57'],\n", + " ['Grand Bahama', 'location.location.nearby_airports', 'West End Airport'],\n", + " ['Help!', 'film.film.language', 'English Language'],\n", + " ['North Andros', 'location.location.containedby', 'Andros, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc3tgyx'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4391d'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4bh36'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1s064hb_b'],\n", + " ['North Eleuthera', 'location.location.containedby', 'Eleuthera'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84kq'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1q5k0l8p_'],\n", + " ['Sovereign state',\n", + " 'base.aareas.schema.administrative_area_type.pertains_to',\n", + " 'Earth'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'Senate of the Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3w9ls'],\n", + " ['Henry Milton Taylor', 'people.person.profession', 'Politician'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblt7'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtx'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Leslie'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc3ltvy'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc46qxk'],\n", + " ['Sloan Farrington', 'people.person.gender', 'Male'],\n", + " ['Bahamas', 'symbols.flag_referent.flag', 'm.0jqg_bs'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3l24m'],\n", + " ['Nassau',\n", + " 'location.location.contains',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'San Salvador and Rum Cay'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0_mv93_'],\n", + " ['Rock Sound International Airport',\n", + " 'location.location.containedby',\n", + " 'Eleuthera'],\n", + " ['Eleuthera', 'location.location.contains', \"Governor's Harbour Airport\"],\n", + " ['1926 Nassau hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Stafford Sands', 'people.person.profession', 'Politician'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.12tb6f6xv'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09w3hrf'],\n", + " ['North America', 'location.location.containedby', 'Americas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3sn1p'],\n", + " ['Bahamas', 'location.location.contains', 'East Grand Bahama'],\n", + " ['1938 New England hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3n9qr'],\n", + " ['m.011cs6y5',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['1899 San Ciriaco hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Geographical Feature',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Non-Agent'],\n", + " ['West End Airport', 'common.topic.notable_types', 'Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1246066qf'],\n", + " ['Treasure Cay', 'common.topic.notable_types', 'Location'],\n", + " [\"Arthur's Town Airport\", 'aviation.airport.serves', \"Arthur's Town\"],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1s064rpqn'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm6n_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4c0b8'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4yx1f'],\n", + " ['Nicholls Town', 'location.location.nearby_airports', 'San Andros Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3p98v'],\n", + " ['m.09wsn92',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc4hvz3'],\n", + " ['m.09yqq63', 'common.webpage.topic', 'Bahamas'],\n", + " ['Spring Point Airport', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3l5l3'],\n", + " ['Time Zone', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0j5n4yk', 'people.marriage.spouse', 'Hubert Ingraham'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'g.11b70mns9b'],\n", + " ['Bahamas',\n", + " 'location.country.first_level_divisions',\n", + " 'Ragged Island, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245_w5fg'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.12cp_jhfq'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6m_v'],\n", + " ['Hurricane Floyd',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84hj'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.124609ngt'],\n", + " ['Alice Town', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.12tb6f6sw'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245ywjj5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4twwr'],\n", + " ['Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'City/Town/Village should pertain to Ghana, since City/Town/Village subdivides Ghanaian Municipal District.'],\n", + " ['West Grand Bahama', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Sandy Creek (Bahamas)'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khltpq'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3zxt3'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'm.0nf5f15'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc38m4x'],\n", + " ['Politician', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc4f8jp'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Mindy'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Joel Stubbs'],\n", + " ['Grand Bahama International Airport',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Patrick Harris'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6g_dr'],\n", + " ['Congo Town', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064sn1l'],\n", + " ['Pleasant Bridgewater', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064v07x'],\n", + " ['Exuma', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245_8rw5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zdyrr'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.04g8ppw'],\n", + " ['Bahamas', 'location.location.contains', 'Paradise Island'],\n", + " ['Teray Smith', 'common.topic.notable_types', 'Athlete'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4z3c1'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4k31p'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84hz'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1q5jby664'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.1hhc3jtwt'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245yz6_j'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Flora'],\n", + " ['m.065pbkf', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6hc0_'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Subtropical Storm Andrea'],\n", + " ['Parliament of the Bahamas',\n", + " 'government.governmental_body.jurisdiction',\n", + " 'Bahamas'],\n", + " ['District of the Bahamas',\n", + " 'base.aareas.schema.administrative_area_type.subdivides_place',\n", + " 'Bahamas'],\n", + " ['Politician', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84jm'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc46r8r'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.12460bz12'],\n", + " ['Bahamas',\n", + " 'location.country.administrative_divisions',\n", + " 'Nichollstown and Berry Islands'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1hhc4xyp_'],\n", + " ['Sandy Creek (Bahamas)', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3hg20'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc3drhj'],\n", + " ['Bahamas', 'location.location.contains', 'San Andros Airport'],\n", + " ['Map of Bahamas',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'New Providence'],\n", + " ['Harbour Island, Bahamas', 'location.location.contains', 'Dunmore Town'],\n", + " ['Demitri Knowles', 'common.topic.notable_types', 'Athlete'],\n", + " ['The Bluff, Bahamas', 'common.topic.notable_types', 'Location'],\n", + " ['Eric Gibson', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4dd26'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc4yc67'],\n", + " ['Hurricane Arthur',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.11b71l32_m'],\n", + " ['Andros, Bahamas', 'location.location.contains', 'Mangrove Cay'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_pg60'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Carol'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Floyd'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xkh'],\n", + " ['North Eleuthera Airport', 'location.location.containedby', 'Eleuthera'],\n", + " ['Eastern Time Zone',\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'g.125_rxy8r'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3tgw4'],\n", + " ['Cyril Stevenson', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Tureano Johnson', 'people.person.nationality', 'Bahamas'],\n", + " ['Jonathan Barry', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Jonathan Barry'],\n", + " ['Tongue of the Ocean', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064swxw'],\n", + " ['Snug Corner', 'common.topic.notable_types', 'Location'],\n", + " ['Abaco Islands', 'location.location.contains', 'Central Abaco'],\n", + " ['Hurricane Dennis',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', \"Deadman's Cay Airport\"],\n", + " [\"Dean's Blue Hole\",\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'Long Island'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_rxzc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4k6hx'],\n", + " ['Eleuthera',\n", + " 'location.location.contains',\n", + " 'Rock Sound International Airport'],\n", + " ['Harbour Island, Bahamas',\n", + " 'common.topic.notable_types',\n", + " 'Geographical Feature'],\n", + " ['Mario Ford', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3j5kh'],\n", + " ['Bahamas',\n", + " 'location.country.languages_spoken',\n", + " 'Bahamas Creole English Language'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3xktk'],\n", + " ['m.09ylvdz', 'common.webpage.topic', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.11b60tqm2w'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84lt'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_pg61'],\n", + " ['Bahamas', 'location.location.contains', 'Atlantis Royal Tower West'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc41kr1'],\n", + " ['Kendal Pinder', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'g.1245z_n91'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_pvq7'],\n", + " ['Bahamas', 'location.location.events', 'Raid on Nassau'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4cnq5'],\n", + " ['Paradise Island', 'common.topic.notable_types', 'Geographical Feature'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbll8'],\n", + " ['Ryan Ingraham', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Hubert Ingraham', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ophelia'],\n", + " ['The Assemblies of God Bible College, Nassau, NP, Bahamas',\n", + " 'education.educational_institution.campuses',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['South Andros Airport', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1q5j7kngh'],\n", + " ['m.0khm7ss', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc3mglw'],\n", + " ['Reginald James Poitier', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3f1mw'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mz2'],\n", + " ['Hurricane Tomas',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Central Abaco', 'location.location.containedby', 'Abaco Islands'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4xn0_'],\n", + " ['Patrick Harris', 'common.topic.notable_types', 'Athlete'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4nfp5'],\n", + " ['District of the Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'City/Town/Village should pertain to Ghana, since City/Town/Village subdivides Ghanaian Municipal District.'],\n", + " ['Hurricane Gert', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc3kkjp'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1935 Labor Day hurricane'],\n", + " ['Bahamas', 'location.location.contains', 'Atlantis Royal Tower East'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3k0q_'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Progressive Liberal Party'],\n", + " ['Grand Bahama International Airport',\n", + " 'location.location.containedby',\n", + " 'Grand Bahama'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6g_ds'],\n", + " ['Queen Victoria Platt', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1q5k0gjbb'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc46qv5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4g98d'],\n", + " ['Patrick Harris', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', \"Dean's Blue Hole\"],\n", + " ['Eastern Time Zone', 'common.topic.article', 'm.0j62c65'],\n", + " ['Pauline Davis-Thompson', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Berry Islands'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6h4yq'],\n", + " ['Andros Town', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.11b71mdmwq'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xb3'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pb_w'],\n", + " ['Andros Town', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4lc3f'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3yf7k'],\n", + " ['Oneil Levy', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3h350'],\n", + " ['1935 Yankee hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblns'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xfc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3fskt'],\n", + " ['Somerset Creek', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas', 'location.location.contains', 'Kemps Bay'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc39kmc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84ll'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'South Andros'],\n", + " ['m.09wsc2h',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Map of Bahamas', 'common.image.appears_in_topic_gallery', 'Fresh Creek'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4wch7'],\n", + " ['Paradise Island', 'geography.island.island_group', 'Lucayan Archipelago'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Queen Victoria Platt'],\n", + " ['Harbour Island, Bahamas',\n", + " 'location.location.contains',\n", + " 'Harbour Island District'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblnj'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Garcha Blair'],\n", + " ['m.065pb_w', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245_9gnx'],\n", + " ['Patrick Harris', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3fgxj'],\n", + " ['Harbour Island, Bahamas',\n", + " 'location.location.time_zones',\n", + " 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mvb'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Godfrey Ellis'],\n", + " ['m.064cvnf', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc3l9dq'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlcs1'],\n", + " ['m.09wsn6r',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Jackner Louis'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3l5l2'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc4l7x1'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4pgdb'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['Bahamas', 'location.location.contains', 'Lucaya, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc51lpd'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4w115'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Damien Neville'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3lmbw'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Inez'],\n", + " ['Americas', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc39p17'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245znq1m'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1924 Cuba hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6f6y_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4zj9c'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4tjyd'],\n", + " ['Bahamas', 'location.location.contains', 'Nassau'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc3vp_9'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4_s6r'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc41k9_'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.12tb6gh26'],\n", + " ['Amara Jones', 'people.person.nationality', 'Bahamas'],\n", + " ['m.0796snn', 'olympics.olympic_medal_honor.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4k9yn'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124601tzl'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc42f69'],\n", + " ['m.09wsc58',\n", + " 'common.webpage.resource',\n", + " \"John Travolta's son dies in the Bahamas\"],\n", + " ['Freeman Barr', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mv_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc3z1p7'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_xk19'],\n", + " ['Bahamas',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['Grand Bahama', 'location.location.geolocation', 'm.02_q_g8'],\n", + " ['m.09w_2bn',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4stjz'],\n", + " ['College of The Bahamas',\n", + " 'internet.top_level_domain_sponsor.domains',\n", + " 'bs'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3sjbg'],\n", + " ['New Bight Airport',\n", + " 'location.location.containedby',\n", + " 'Cat Island, Bahamas'],\n", + " ['North America', 'base.locations.continents.countries_within', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Dominic Demeritte'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mwn'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xf3'],\n", + " ['Bahamas', 'location.location.people_born_here', 'James Eugene Brown'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Bill'],\n", + " ['m.0khlx9w', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['m.0j1bz6l', 'common.webpage.topic', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3jtqn'],\n", + " ['Bahamas national football team', 'sports.sports_team.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.12tb6gtg8'],\n", + " ['Senate of the Bahamas',\n", + " 'government.governmental_body.members',\n", + " 'm.0_mv93_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xd3'],\n", + " ['Bahamas', 'location.location.contains', 'Stella Maris Airport'],\n", + " ['Bahamas', 'location.location.contains', \"Arthur's Town Airport\"],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc43vbc'],\n", + " ['Dereck Gittens', 'people.person.gender', 'Male'],\n", + " ['m.0645m6v', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Earth', 'base.locations.planets.continents_within', 'North America'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Cleo'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc44s9f'],\n", + " ['Black Point, Bahamas',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3d643'],\n", + " ['Paradise Island',\n", + " 'location.location.contains',\n", + " 'Atlantis Royal Tower West'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc41w_j'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3yfmq'],\n", + " ['District of the Bahamas',\n", + " 'base.aareas.schema.administrative_area_type.sovereignty',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_7s_2'],\n", + " ['Dereck Gittens', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1899 San Ciriaco hurricane'],\n", + " ['Atlantis Royal Tower West', 'location.location.containedby', 'Bahamas'],\n", + " ['1932 Bahamas hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Snug Corner', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4jpth'],\n", + " ['Hurricane Sandy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.12460wzyx'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3r_sd'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc51hjl'],\n", + " ['Damien Neville', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['Henry Milton Taylor', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3cc0b'],\n", + " ['Ragged Island, Bahamas',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_lf14'],\n", + " ['Hurricane Ike', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Sandy Point', 'location.administrative_division.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3xpc5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.12cp_jr3t'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khl8zy'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlrs4'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.0650f8z'],\n", + " ['Time Zone', 'freebase.type_profile.strict_included_types', 'Inanimate'],\n", + " ['Roderick Bowe', 'people.person.nationality', 'Bahamas'],\n", + " ['San Salvador and Rum Cay', 'location.location.containedby', 'Bahamas'],\n", + " ['East Grand Bahama', 'location.location.containedby', 'Bahamas'],\n", + " ['Ragged Island, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['Sapodilla Creek', 'geography.river.basin_countries', 'Bahamas'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09wsn92'],\n", + " ['Commonwealth of the Bahamas Trade Union Congress',\n", + " 'organization.organization.geographic_scope',\n", + " 'Bahamas'],\n", + " ['Stafford Sands', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc51dgt'],\n", + " ['Mount Creek', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09yqq63'],\n", + " ['Eleuthera',\n", + " 'location.location.nearby_airports',\n", + " 'North Eleuthera Airport'],\n", + " ['1941 Florida hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['m.0khm15p', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Rum Cay District'],\n", + " [\"Dean's Blue Hole\", 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc51_jv'],\n", + " ['Bahamas', 'organization.organization_member.member_of', 'm.04kc4ps'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xg7'],\n", + " ['Hurricane Cleo', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Coalition for Democratic Reform'],\n", + " ['Map of Bahamas', 'common.image.appears_in_topic_gallery', 'Grand Bahama'],\n", + " ['Lucayan Archipelago', 'location.location.contains', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6flvl'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.0645m6v'],\n", + " ['Acklins Island', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3qg5c'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4_k3h'],\n", + " ['Hurricane Isabel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pbcr'],\n", + " ['Demitri Knowles', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Sloan Farrington'],\n", + " ['District of the Bahamas',\n", + " 'freebase.type_hints.included_types',\n", + " 'Location'],\n", + " ['Speed 2: Cruise Control', 'film.film.language', 'English Language'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zhqqp'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84h2'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mww'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4krzp'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1941 Florida hurricane'],\n", + " ['Ragged Island, Bahamas',\n", + " 'location.administrative_division.country',\n", + " 'Bahamas'],\n", + " ['m.064thwt', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc4j_wd'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3tp1f'],\n", + " ['Hubert Ingraham', 'common.topic.article', 'm.0367nh'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4xvm3'],\n", + " ['Bahamas', 'location.location.containedby', 'Americas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3g2xk'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3nx1d'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Inagua'],\n", + " ['Simone Pratt', 'people.person.place_of_birth', 'Grand Bahama'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4krzn'],\n", + " ['Eleuthera', 'location.location.contains', 'North Eleuthera Airport'],\n", + " ['Bahamas', 'location.location.contains', 'Marsh Harbour'],\n", + " ['Bahamas', 'sports.sport_country.athletic_performances', 'm.0b652s2'],\n", + " ['Gary Armstrong', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Anya Watkins'],\n", + " ['Treasure Cay Airport', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4hchz'],\n", + " ['m.0_z56t6', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['South Andros', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Tropical Storm Felice',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['m.04cn1xb', 'organization.organization_membership.member', 'Bahamas'],\n", + " ['High Rock', 'location.administrative_division.country', 'Bahamas'],\n", + " ['Time Zone', 'freebase.type_profile.published', 'Published'],\n", + " ['Andros, Bahamas', 'location.location.contains', 'North Andros'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0kd9w_5'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3d68j'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Laverne Eve'],\n", + " ['Bahamas',\n", + " 'location.country.administrative_divisions',\n", + " 'Acklins and Crooked Islands'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245zj913'],\n", + " ['Ron Butler', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4s1qt'],\n", + " ['Stella Maris Airport', 'aviation.airport.serves', 'Long Island'],\n", + " ['Battle of Nassau', 'time.event.locations', 'Nassau'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3zxp6'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mx2'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlf_8'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.12cp_h_jb'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4m8g2'],\n", + " ['Bahama Banks', 'location.location.containedby', 'Bahamas'],\n", + " ['m.0cpn7wp',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.country.form_of_government', 'Parliamentary system'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4tfns'],\n", + " ['Freeman Barr', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1q5k9vr8j'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Donald Knowles'],\n", + " ['Andrew Ford', 'people.person.gender', 'Male'],\n", + " [\"John Travolta's son dies in the Bahamas\",\n", + " 'common.resource.annotations',\n", + " 'm.09wsn92'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4dy6r'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pd9q'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc3qww1'],\n", + " ['Long Island', 'location.location.nearby_airports', 'Stella Maris Airport'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3d9kd'],\n", + " ['Grand Bahama', 'common.topic.image', 'Grand Bahama from space, June 1998'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc4gvdw'],\n", + " ['Bahamas', 'location.location.contains', 'Somerset Creek'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc455tv'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Diana'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc38v52'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gordon'],\n", + " ['Marsh Harbour Airport', 'aviation.airport.serves', 'Marsh Harbour'],\n", + " ['Map of Bahamas', 'common.image.size', 'm.029grh2'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblks'],\n", + " ['East Grand Bahama',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064ggl7'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1hhc4b264'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3cl13'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245z86qy'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4wcgc'],\n", + " ['Patrick Harris', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.12cp_k2sl'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblp8'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064sm0y'],\n", + " [\"Walker's Cay\", 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc40r9_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_jzfx'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Edith'],\n", + " ['Coopers Town', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'm.0nf5f1m'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mxz'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_kjqp'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zmyst'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pbkf'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc3yfcz'],\n", + " ['Inagua', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Winston Saunders'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Fox'],\n", + " ['m.0khm2q6', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Reginald James Poitier', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc40yr8'],\n", + " ['Location', 'freebase.type_profile.published', 'Published'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3czyk'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4nnfx'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84hr'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n34'],\n", + " ['Lucayan Archipelago', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc5541g'],\n", + " ['Bahamas', 'location.location.contains', 'Black Point, Bahamas'],\n", + " ['Amara Jones', 'olympics.olympic_athlete.country', 'm.0khlsrg'],\n", + " ['Black Point, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtg'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3_pmw'],\n", + " ['Airport', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlsrg'],\n", + " ['Alvin Smith', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1hhc3p648'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.brain_drain_percent',\n", + " 'g.1hhc4t_s6'],\n", + " [\"John Travolta's son dies in the Bahamas\",\n", + " 'common.resource.annotations',\n", + " 'm.09wsc2h'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84kh'],\n", + " ['William Cartwright',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'Progressive Liberal Party'],\n", + " ['m.065pd9q', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Coopers Town'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3lm8k'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbllj'],\n", + " ['Somerset Creek', 'location.location.containedby', 'Bahamas'],\n", + " ['Gregory Taylor', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc4w0_t'],\n", + " ['Bahamas', 'location.location.contains', 'New Providence'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n3c'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mzr'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.11b71n53gy'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84ky'],\n", + " ['Bahamas',\n", + " 'location.country.first_level_divisions',\n", + " 'Green Turtle Cay District'],\n", + " ['Gregory Taylor', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['m.03hc_rb', 'common.webpage.topic', 'Eastern Time Zone'],\n", + " ['m.0khltpq', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Marvin Rolle'],\n", + " ['m.07w429x',\n", + " 'olympics.olympic_medal_honor.olympics',\n", + " '2008 Summer Olympics'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3dvtf'],\n", + " ['Mario Ford', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc44z_x'],\n", + " ['Bahamas', 'location.country.administrative_divisions', 'New Providence'],\n", + " ['m.09x2k1q',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.11b60wz9rq'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3tknp'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3w6kn'],\n", + " [\"Arthur's Town Airport\",\n", + " 'location.location.containedby',\n", + " 'Cat Island, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4p1tb'],\n", + " ['Athlete',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Physically Instantiable'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.12tb6ggym'],\n", + " ['Bahamas', 'location.location.contains', 'Harbour Island, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc3pmrq'],\n", + " ['Bahamas', 'location.country.administrative_divisions', 'High Rock'],\n", + " ['Airport', 'freebase.type_profile.published', 'Published'],\n", + " ['South Abaco',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4hc7p'],\n", + " ['Kemps Bay', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Emily'],\n", + " ['1935 Labor Day hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Tropical Storm Mindy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8x9m'],\n", + " ['Bahamas', 'location.country.first_level_divisions', \"Moore's Island\"],\n", + " ['Hurricane Hugo', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4s847'],\n", + " ['m.0b64vms',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'Bahamas'],\n", + " ['New Providence', 'location.administrative_division.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4s8d0'],\n", + " ['Marvin Rolle', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['San Salvador Island', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblmj'],\n", + " ['Andros Town International Airport',\n", + " 'aviation.airport.serves',\n", + " 'Andros, Bahamas'],\n", + " ['Map of Bahamas', 'common.image.appears_in_topic_gallery', 'Inagua'],\n", + " ['Juan Lewis', 'common.topic.notable_types', 'Athlete'],\n", + " ['Cat Island, Bahamas', 'location.location.contains', 'New Bight Airport'],\n", + " ['Bahamas', 'location.location.contains', 'Tarpum Bay'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Michelle'],\n", + " ['Bahamas', 'location.location.contains', 'Grand Cay'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.12460ghrf'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1s067rs_9'],\n", + " ['Hurricane Hazel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.0j1bz6l'],\n", + " ['m.0khlz57', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4ysk3'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.1hhc3dz9l'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.124611gb8'],\n", + " ['Ulrich Fox', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Geographical Feature', 'type.type.domain', 'Physical Geography'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.12460_765'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n3l'],\n", + " ['Tropical Storm Dean',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Leeward Islands Airline Pilots Association',\n", + " 'organization.organization.geographic_scope',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Forfar Field Station, Bahamas'],\n", + " ['Bahamas', 'location.country.administrative_divisions', 'Marsh Harbour'],\n", + " ['Green Turtle Cay', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245ytnvz'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Long Island'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Eric Gibson'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3_s_3'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064hd12'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.010qympf'],\n", + " ['Bahamas', 'location.location.contains', 'Bimini'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Franklin'],\n", + " ['Mount Creek', 'common.topic.notable_types', 'Location'],\n", + " ['Tureano Johnson', 'people.person.gender', 'Male'],\n", + " ['Long Island',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'River Lees'],\n", + " ['Evelyn Poitier', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1246098gb'],\n", + " ['Bahamas', 'location.location.contains', 'Marsh Harbour Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4q8qf'],\n", + " ['m.0khm4hg', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Juan Lewis', 'people.person.place_of_birth', 'Grand Bahama'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblsh'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n1d'],\n", + " ['Paradise Island',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'Nassau'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.12tb6f_06'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245ytnhr'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc51x2t'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n01'],\n", + " ['Country', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Fabienne Dali'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Free National Movement'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4dd43'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1926 Nassau hurricane'],\n", + " ['Hurricane Hanna',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1s064hbh1'],\n", + " ['Bennet Davis', 'people.person.gender', 'Male'],\n", + " ['Andros Town International Airport',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khly48'],\n", + " ['Rico Forbes', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4gvqm'],\n", + " ['New Providence', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm15p'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245z78xl'],\n", + " ['m.064ssr7', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3m_rc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.1hhc3tkpr'],\n", + " ['m.0j5n49m',\n", + " 'government.government_position_held.office_holder',\n", + " 'Hubert Ingraham'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbls7'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3pmqr'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc4w10b'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3y9n5'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1928 Okeechobee hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.11b71wcvmf'],\n", + " ['Grand Bahama', 'location.location.containedby', 'Bahamas'],\n", + " [\"Arthur's Town\",\n", + " 'location.location.nearby_airports',\n", + " \"Arthur's Town Airport\"],\n", + " ['Reginald James Poitier', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['The Cove Atlantis', 'location.location.containedby', 'Bahamas'],\n", + " ['Long Island', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Matthew Town'],\n", + " ['Casa-bahama', 'common.image.appears_in_topic_gallery', 'Grand Bahama'],\n", + " ['Hope Town', 'location.location.containedby', 'Abaco Islands'],\n", + " ['Bahamas', 'organization.organization_member.member_of', 'm.04cn1xb'],\n", + " ['Rock Sound International Airport',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['Rosanna Carter', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Vanguard Nationalist and Socialist Party'],\n", + " ['Bahamas', 'location.location.contains', 'g.11x7vfs1t'],\n", + " ['m.064swxw', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Robert Eardley', 'people.person.gender', 'Male'],\n", + " ['Sandy Point', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc41w_t'],\n", + " ['Treasure Cay', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Arthur'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblp0'],\n", + " ['Berry Islands', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245_xkfc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xg_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4sh4p'],\n", + " ['Bahamas', 'location.location.contains', 'Ragged Island, Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1926 Miami hurricane'],\n", + " ['Andrew Ford', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc37hbf'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zbnd8'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Pauline Davis-Thompson'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4s853'],\n", + " ['Hurricane Wilma',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Gale of 1878'],\n", + " ['m.0khlv1n', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Dunmore Town', 'location.location.containedby', 'Bahamas'],\n", + " ['m.0khl8zy', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbls_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1q5jpwtpx'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Ginny'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1s06b0p6_'],\n", + " ['Dwight Weakley', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xdw'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm1yz'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.124607lb6'],\n", + " ['Fabienne Dali', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3gq61'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3_sv3'],\n", + " ['Hurricane Bertha',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['1948 Miami hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Gert'],\n", + " ['Exuma',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Abaco Islands', 'common.topic.notable_types', 'Geographical Feature'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Hubert Minnis'],\n", + " ['James Eugene Brown', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Dunmore Town'],\n", + " ['Bahamas',\n", + " 'base.ontologies.ontology_instance.equivalent_instances',\n", + " 'm.07ndmxt'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.11b71sl3km'],\n", + " ['Colonel Hill', 'location.location.containedby', 'Bahamas'],\n", + " ['Berry Islands', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Rico Forbes', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc47bf9'],\n", + " ['Coopers Town', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.011cs6y5'],\n", + " ['James Eugene Brown', 'people.person.nationality', 'Bahamas'],\n", + " ['Hurricane Betsy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc41cb0'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Fay'],\n", + " ['Bahamas', 'location.location.contains', 'Snug Corner'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3zg_f'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc449nk'],\n", + " ['Laverne Eve', 'people.person.nationality', 'Bahamas'],\n", + " ['m.09wsc58', 'common.webpage.topic', 'Bahamas'],\n", + " ['San Salvador and Rum Cay',\n", + " 'location.administrative_division.country',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4ts5v'],\n", + " ['Grand Bahama', 'location.location.people_born_here', 'Hubert Ingraham'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xj_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3szfr'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblr7'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Acklins and Crooked Islands',\n", + " 'location.administrative_division.country',\n", + " 'Bahamas'],\n", + " ['m.09x2k1q', 'common.webpage.topic', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Mount Creek'],\n", + " ['Bahamas', 'location.location.contains', 'Alice Town'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc40jvz'],\n", + " ['m.0khlf_8', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['m.064sml9', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3pym2'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gracie'],\n", + " ['Sandy Creek (Bahamas)', 'location.location.containedby', 'Bahamas'],\n", + " ['Sovereign state',\n", + " 'base.aareas.administrative_area_level.examples',\n", + " 'Sovereign state'],\n", + " ['Parliament of the Bahamas',\n", + " 'government.governmental_body.component_bodies',\n", + " 'Senate of the Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4qq41'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc40ywy'],\n", + " ['m.0_mv93_',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4gqnx'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.12460yhx1'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n2y'],\n", + " ['Tropical Storm Gilda',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['River Lees', 'location.location.containedby', 'Bahamas'],\n", + " ['m.0khm1yz', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Grand Bahama International Airport',\n", + " 'aviation.airport.hub_for',\n", + " 'Western Air'],\n", + " ['San Salvador Island',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Bertha'],\n", + " ['Andros Town', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Wilma'],\n", + " ['Shonel Ferguson', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245_m5vm'],\n", + " ['Bahamas',\n", + " 'location.country.administrative_divisions',\n", + " 'Green Turtle Cay District'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc530zb'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc4gqqv'],\n", + " ['Crooked Island, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['Spanish Wells', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['West Grand Bahama',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Juan Lewis', 'common.topic.article', 'm.0gj9yx8'],\n", + " ['Eleuthera', 'location.location.contains', 'Central Eleuthera'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Arlene'],\n", + " ['Jonathan Barry', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc40rcg'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['Geographical Feature',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Physically Instantiable'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblps'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc53l3f'],\n", + " ['New Providence', 'location.location.contains', 'College of The Bahamas'],\n", + " ['Paradise Island', 'location.location.contains', 'The Cove Atlantis'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1932 Cuba hurricane'],\n", + " ['Gilbert NMO Morris', 'people.person.gender', 'Male'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Teray Smith'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4mtxm'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Clinton Outten'],\n", + " ['Nichollstown and Berry Islands',\n", + " 'location.administrative_division.country',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xcw'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Oneil Levy'],\n", + " ['Democratic National Alliance',\n", + " 'organization.organization.geographic_scope',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.country.national_anthem', 'm.0d8ndb_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xfm'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245zkdm9'],\n", + " ['Nicholls Town', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_q10k'],\n", + " ['Nassau', 'location.location.events', 'Battle of Nassau'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245_j1px'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Dennis'],\n", + " ['1933 Treasure Coast hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'base.athletics.athletics_country.championships_athletes_performances',\n", + " 'm.06dn7m0'],\n", + " ['North Eleuthera Airport', 'location.location.containedby', 'Bahamas'],\n", + " ['Alice Town', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Exuma'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.12460cqfl'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Whitcliff Atkinson'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4ts64'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc3d68w'],\n", + " ['Starve Creek', 'common.topic.notable_types', 'Location'],\n", + " ['Starve Creek', 'geography.river.basin_countries', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0j5n49m'],\n", + " ['Andrew Ford', 'people.person.nationality', 'Bahamas'],\n", + " ['Cat Island, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Dwight Weakley'],\n", + " ['Hurricane Edna', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Tom Robinson', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc4469t'],\n", + " ['1947 Fort Lauderdale hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc4f7ny'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.12460n65r'],\n", + " ['Nassau',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Paradise Island'],\n", + " ['Central Eleuthera', 'location.location.containedby', 'Eleuthera'],\n", + " ['Grand Bahama International Airport', 'common.topic.article', 'm.07jwhc'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblqy'],\n", + " ['Eastern Time Zone', 'common.topic.image', 'UnionSquareAtomicClock'],\n", + " ['Ryan Ingraham', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mzj'],\n", + " ['Green Turtle Cay District',\n", + " 'location.location.containedby',\n", + " 'Green Turtle Cay'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3pyb9'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.12460w01w'],\n", + " ['Bahamas', 'location.country.administrative_divisions', 'Long Island'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mz9'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3kwxf'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Tureano Johnson'],\n", + " ['Tarpum Bay', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas',\n", + " 'location.location.events',\n", + " 'Spain in the American Revolutionary War'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc46r09'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Central Andros'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'm.0nf5f1d'],\n", + " ['1891 Martinique hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbln0'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlpp9'],\n", + " ['Bahamas',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['The Cove Atlantis', 'location.location.containedby', 'Paradise Island'],\n", + " ['Estelle Evans', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtp'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4vm01'],\n", + " ['Long Island',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " \"Dean's Blue Hole\"],\n", + " ['Gary Armstrong', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bahamas', 'location.location.contains', 'Sandy Point'],\n", + " ['Bahamas Ministry of Foreign Affairs',\n", + " 'government.government_agency.jurisdiction',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.country.administrative_divisions',\n", + " 'San Salvador and Rum Cay'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4l7y9'],\n", + " ['UTC−05:00', 'common.topic.notable_types', 'Time Zone'],\n", + " ['Whitcliff Atkinson', 'people.person.gender', 'Male'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.124602dgk'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Sandy'],\n", + " ['Hurricane Isbell',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khlcfh'],\n", + " ['Bahamas', 'location.location.contains', 'Freetown, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc403l0'],\n", + " ['Geographical Feature', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124607cj7'],\n", + " ['Atlantis Royal Tower East', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1949 Florida hurricane'],\n", + " ['m.04g8ppw',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['Spanish Wells',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4vmc8'],\n", + " ['Spring Point Airport', 'common.topic.notable_types', 'Airport'],\n", + " ['New Bight Airport', 'common.topic.notable_types', 'Airport'],\n", + " ['Blake Bartlett', 'common.topic.notable_types', 'Athlete'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pc4_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6myw'],\n", + " ['Hurricane Lili', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3z8cs'],\n", + " ['Bahamas',\n", + " 'base.aareas.schema.administrative_area.subdividing_type',\n", + " 'District of the Bahamas'],\n", + " ['Hubert Ingraham', 'people.person.gender', 'Male'],\n", + " ['Simon Creek', 'location.location.containedby', 'Bahamas'],\n", + " ['m.09wsn6r',\n", + " 'common.webpage.resource',\n", + " \"John Travolta's son dies in the Bahamas\"],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245z9q9c'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Ike'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Edna'],\n", + " ['Cat Island, Bahamas', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Grand Bahama', 'common.topic.article', 'm.03st9s'],\n", + " ['Henry Milton Taylor', 'people.person.gender', 'Male'],\n", + " ['Hubert Ingraham',\n", + " 'government.politician.government_positions_held',\n", + " 'm.0j5n49m'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xhh'],\n", + " ['Bahamas',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Sovereign state'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6gh39'],\n", + " ['Hubert Ingraham', 'people.person.profession', 'Politician'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n0y'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc52mnm'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblpj'],\n", + " [\"Deadman's Cay Airport\", 'common.topic.notable_types', 'Airport'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblq1'],\n", + " ['Pauline Davis-Thompson',\n", + " 'olympics.olympic_athlete.medals_won',\n", + " 'm.07fc00l'],\n", + " ['Hurricane Emily',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Amara Jones'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc51px2'],\n", + " ['m.0d8ndb_', 'government.national_anthem_of_a_country.country', 'Bahamas'],\n", + " ['Nichollstown and Berry Islands',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['Teray Smith', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Spring Point Airport'],\n", + " ['Winston Saunders', 'people.person.gender', 'Male'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Anita Allen'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3f_cn'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbln8'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1s067k989'],\n", + " ['Bahamas', 'location.location.contains', 'Long Island'],\n", + " ['Bahamas', 'film.film_location.featured_in_films', 'Casino Royale'],\n", + " ['Marsh Harbour', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc5431y'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.12cp_jcl1'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'Parliament of the Bahamas'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Location'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Albert Roker'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6flvm'],\n", + " ['Clinton Outten', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc488yy'],\n", + " ['Holiday in the Sun', 'film.film.featured_film_locations', 'Bahamas'],\n", + " ['Jimmy Spice Curry', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Sloan Farrington', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Berry Islands',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Rita'],\n", + " ['Bahamas', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['Casa-bahama', 'common.image.size', 'm.04p9wvb'],\n", + " ['1919 Florida Keys hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Acklins'],\n", + " ['River Lees', 'geography.river.basin_countries', 'Bahamas'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Llewellyn Armstrong'],\n", + " ['m.064smzc', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['m.0vskksj', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'sports.sport_country.athletic_performances', 'm.0b64vms'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6myn'],\n", + " ['Eastern Time Zone', 'common.topic.article', 'm.0k83440'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc52vkr'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_mywj'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4dlq2'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4cp03'],\n", + " ['Bahamas', 'location.location.contains', 'Spanish Wells'],\n", + " ['Bahamas', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4q8kh'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc4cbr3'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4f7vc'],\n", + " ['Harbour Island District',\n", + " 'location.location.containedby',\n", + " 'Harbour Island, Bahamas'],\n", + " ['Grand Bahama International Airport',\n", + " 'common.topic.notable_types',\n", + " 'Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'g.1245_d2kc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4d8mz'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Commonwealth of the Bahamas Trade Union Congress'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Lili'],\n", + " ['Hubert Ingraham', 'people.person.children', 'Kelly Ingraham'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc3_4lr'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3vpty'],\n", + " ['Geographical Feature',\n", + " 'freebase.type_profile.equivalent_topic',\n", + " 'Geographical feature'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc5513s'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.12460fd8z'],\n", + " ['Hurricane Gracie',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ...],\n", + " [['Hurricane Betsy', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Isabel'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.12460kjms'],\n", + " ['Hurricane Flora',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'New Jersey'],\n", + " ['Cat Island, Bahamas', 'common.topic.notable_for', 'g.1255jz93k'],\n", + " ['Tarpum Bay', 'common.topic.article', 'm.02vpd3x'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'North Carolina'],\n", + " ['Dominica', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Hugo'],\n", + " ['m.0khl4t6', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc3fsdd'],\n", + " ['Sidney Poitier',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['East Grand Bahama', 'common.topic.article', 'm.070vd9'],\n", + " ['Haiti',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Fay'],\n", + " ['Estelle Evans', 'people.person.profession', 'Actor'],\n", + " ['m.064ggl7', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Mississippi',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['Paradise Island',\n", + " 'location.location.contains',\n", + " 'Atlantis Royal Tower East'],\n", + " ['Central America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Lili'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc41kjl'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc3fspl'],\n", + " ['North Eleuthera', 'common.topic.article', 'm.0bkcr0'],\n", + " ['m.0khlv1n',\n", + " 'olympics.olympic_athlete_affiliation.sport',\n", + " 'Track and field athletics'],\n", + " ['Hurricane Dennis',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Hurricane Donna', 'meteorology.tropical_cyclone.affected_areas', 'Haiti'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Greta'],\n", + " ['North Carolina',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2000 Summer Olympics'],\n", + " ['Hurricane Flora', 'meteorology.tropical_cyclone.affected_areas', 'Cuba'],\n", + " ['Hurricane Cleo', 'meteorology.tropical_cyclone.affected_areas', 'Florida'],\n", + " ['Hurricane Wilma', 'meteorology.tropical_cyclone.affected_areas', 'Cuba'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1919 Florida Keys hurricane'],\n", + " ['Perry Christie', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.11b71l32_5'],\n", + " ['1941 Florida hurricane',\n", + " 'common.topic.image',\n", + " '1941 Florida hurricane Daily Weather Map'],\n", + " ['m.0796snn', 'olympics.olympic_medal_honor.medal', 'Gold medal'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3hg97'],\n", + " ['Johnny Depp', 'people.person.gender', 'Male'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'New Jersey'],\n", + " ['Donald Knowles', 'common.topic.notable_types', 'Deceased Person'],\n", + " ['College/University', 'freebase.type_hints.included_types', 'Organization'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Harbour Island'],\n", + " ['West Virginia', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Kate Bosworth', 'people.person.gender', 'Female'],\n", + " ['Hurricane Edna',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['Hurricane Paloma',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Louisiana',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['Bahamas', 'location.location.contains', 'Mayaguana'],\n", + " ['m.0w9cvgd',\n", + " 'soccer.football_player_stats.team',\n", + " 'Bahamas national football team'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'Bahamas Ministry of Foreign Affairs'],\n", + " ['Bahamas', 'location.location.contains', 'Bahama Banks'],\n", + " ['North Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1935 Labor Day hurricane'],\n", + " ['Warren Fraser', 'people.person.gender', 'Male'],\n", + " ['Bahamas Hotel, Catering and Allied Workers Union',\n", + " 'common.topic.notable_for',\n", + " 'g.1257l3xjd'],\n", + " ['Kate Bosworth', 'base.popstra.celebrity.vacations_in', 'm.064smzc'],\n", + " ['The Assemblies of God Bible College, Nassau, NP, Bahamas',\n", + " 'education.educational_institution_campus.educational_institution',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['South Florida',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['Marina Moore', 'people.person.gender', 'Female'],\n", + " ['Laverne Eve', 'common.topic.article', 'm.0dk8zy'],\n", + " ['North Andros', 'location.statistical_region.population', 'g.11bc85v_d9'],\n", + " ['Dominica', 'common.topic.notable_types', 'Country'],\n", + " ['m.0jqg_bs', 'symbols.flag_use.flag_user', 'Bahamas'],\n", + " ['Hurricane Noel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc431x7'],\n", + " ['Abali Hoilett', 'common.topic.article', 'm.0hrbdbj'],\n", + " ['Hurricane Isbell',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['m.0n3k846',\n", + " 'government.government_position_held.office_position_or_title',\n", + " 'Prime Minister of the Bahamas'],\n", + " ['Help!', 'film.film.language', 'English Language'],\n", + " ['Jackner Louis', 'sports.pro_athlete.teams', 'm.0j3wyf_'],\n", + " ['1919 Florida Keys hurricane',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1919 Atlantic hurricane season'],\n", + " ['Ohio', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Frances'],\n", + " ['Tom Robinson', 'people.person.education', 'm.0667qsh'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc3tgyx'],\n", + " ['Governor-General of the Bahamas',\n", + " 'government.government_office_or_title.category',\n", + " 'Governor-General'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.time_required_to_start_a_business',\n", + " 'g.1s064hb_b'],\n", + " ['Sovereign state',\n", + " 'base.aareas.schema.administrative_area_type.pertains_to',\n", + " 'Earth'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtx'],\n", + " ['Fabienne Dali', 'film.actor.film', 'm.0xntgpw'],\n", + " ['Puerto Rico',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1919 Florida Keys hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3l24m'],\n", + " ['Canada',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'City/Town/Village should pertain to Ghana, since City/Town/Village subdivides Ghanaian Municipal District.'],\n", + " ['Reginald James Poitier', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0_mv93_'],\n", + " ['m.0wh_2dv', 'sports.sports_team_roster.player', 'Demitri Knowles'],\n", + " ['Demetrius Pinder', 'people.person.nationality', 'Bahamas'],\n", + " ['Currency', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Rock Sound International Airport',\n", + " 'location.location.containedby',\n", + " 'Eleuthera'],\n", + " ['Puerto Rico',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gert'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Hugo'],\n", + " ['Garcha Blair', 'common.topic.article', 'm.0c0307b'],\n", + " ['Bimini', 'common.topic.image', 'Bimini island'],\n", + " ['m.0nfblq1',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['North America', 'location.location.containedby', 'Americas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3sn1p'],\n", + " ['Bahamas', 'location.location.contains', 'East Grand Bahama'],\n", + " ['Hurricane Betsy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['Americas', 'location.location.contains', 'Guadeloupe'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3n9qr'],\n", + " ['m.011cs6y5',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['m.0j7r7th', 'sports.sports_team_roster.player', 'Dwight Weakley'],\n", + " ['Long Island',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Cape Santa Maria Beach'],\n", + " ['Georgia',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Texas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Felice'],\n", + " ['Rico Forbes', 'sports.pro_athlete.teams', 'm.0h_fpst'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3p98v'],\n", + " ['Acklins Island',\n", + " 'location.location.people_born_here',\n", + " 'Arthur Dion Hanna'],\n", + " ['m.0nf8xcc',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['m.010qympf',\n", + " 'government.government_position_held.office_holder',\n", + " 'Arthur Dion Hanna'],\n", + " ['Barbados',\n", + " 'location.country.form_of_government',\n", + " 'Constitutional monarchy'],\n", + " ['Lesser Antilles', 'location.location.contains', 'Windward Islands'],\n", + " ['Hurricane Sandy',\n", + " 'meteorology.tropical_cyclone.category',\n", + " 'Category 3 Hurricane (SSHS)'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'g.11b70mns9b'],\n", + " ['Human Language', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " [\"2009 World Championships in Athletics – Women's 200 metres\",\n", + " 'sports.tournament_event_competition.competitors',\n", + " 'm.0b64vms'],\n", + " ['Central United States',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6m_v'],\n", + " ['South Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Tammy'],\n", + " ['m.011cs2fz',\n", + " 'government.government_position_held.office_holder',\n", + " 'Perry Christie'],\n", + " ['South Abaco', 'common.topic.notable_for', 'g.125bb5r7l'],\n", + " ['Alice Town', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4twwr'],\n", + " ['West Grand Bahama', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Sandy Creek (Bahamas)'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khltpq'],\n", + " ['Massachusetts',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Dominican Republic', 'location.location.containedby', 'Americas'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['District of the Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'Herisau should be contained in some Scottish district, German rural district, Beninese sub-prefectures, Afghan district, East Timorese subdistrict, Belizean constituency, Bangladeshi district, New Zealand district, Seychellois district, Burmese district, Parish of the Isle of Man, Lebanese county, Ghanaian metropolitan district, Thai district, South Korean county, District of the Philippines, Nigerian local government area, Congolese zone, Cape Verdian parish, Papua New Guinean district, Jordanian sub-district, District of Saint Helena, Omani district, Israeli subdistrict, Lithuanian district municipality, Canton of Luxembourg, County of Croatia, Dominican parish, Spanish comarca, Libyan district, District of the Bahamas, Cambodian district, District of the French Southern Territories, District of São Tomé and Príncipe, Paraguayan district, District of Mauritius, Bruneian mukim, Commune of the Faroe Islands, Swiss district, Ghanaian district, Austrian political district, South African district, District of Nauru, German urban district, Slovak district, Polish county, Brazilian municipality, Northern Irish district, Albanian bashkia, Central African sub-prefecture, Hungarian urban county, Canadian county, District of Sierra Leone, Congolese district, Belarusian district, Romanian county, Subdistrict of Botswana, Ethiopian zone, Tanzanian district, County of South Sudan, Belgian arrondissement, Irish county, Rwandan district, District of Hong Kong, Indian subdivision, District of Anguilla, Moldovan district, Surinamese ressort, Syrian district, Cueillete of Jersey, Liberian district, District of Serbia, Somalian district, Kenyan district, Ghanaian ordinary district, Nepalese district, District of Laos, Iranian county, Cayman Islands district, Vingtaine of Jersey, Quarter of Saint Lucia, Angolan council, Gambian district, Singapore district, Guinean subprefecture, Malaysian district, Taiwanese district, Panamanian district, District of Kyrgyzstan, Albanian komuna, Czech district, Indonesian subdistrict, Sudanese district, District of Mozambique, Samoan district, Argentinian department, District of Malawi, Local council of Malta, Ecuadorean canton, Costa Rican canton, or Ghanaian municipal district within Appenzell Ausserrhoden.'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Alabama'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Joel Stubbs'],\n", + " ['Hurricane Rita', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['1924 Cuba hurricane',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1924 Atlantic hurricane season'],\n", + " ['Henry Milton Taylor', 'common.topic.article', 'm.026wxq8'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.04g8ppw'],\n", + " ['Hurricane Frances', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Boxer', 'freebase.type_hints.included_types', 'Athlete'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4k31p'],\n", + " ['m.0nfblq9',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.1hhc3jtwt'],\n", + " ['Hurricane Betsy',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1965 Atlantic hurricane season'],\n", + " ['m.0nf84ky',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['1926 Nassau hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['District of the Bahamas',\n", + " 'freebase.type_hints.included_types',\n", + " 'Administrative Area'],\n", + " ['Honduras',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['Bimini', 'location.location.geolocation', 'm.02_mpz6'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3hg20'],\n", + " ['Football player', 'freebase.type_hints.included_types', 'Person'],\n", + " ['Bahamas', 'location.location.contains', 'San Andros Airport'],\n", + " ['Map of Bahamas',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'New Providence'],\n", + " ['Harbour Island, Bahamas', 'location.location.contains', 'Dunmore Town'],\n", + " ['Tropical Storm Felice',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Louisiana'],\n", + " ['Hurricane Michelle',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '2001 Atlantic hurricane season'],\n", + " ['Hurricane Arthur',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['British Virgin Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Frances'],\n", + " ['The London 2012 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['San Salvador Island', 'location.location.geolocation', 'm.02_n_gc'],\n", + " ['Hurricane Hanna', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Human Language', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Puerto Rico', 'location.country.official_language', 'English Language'],\n", + " ['Atlantic Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Flora'],\n", + " ['Bahamas Taxi Cab Union', 'common.topic.notable_types', 'Organization'],\n", + " ['West Grand Bahama', 'location.location.geolocation', 'm.0cl8whf'],\n", + " ['Bahamas', 'location.location.contains', \"Deadman's Cay Airport\"],\n", + " ['1945 Homestead hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " [\"Dean's Blue Hole\",\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'Long Island'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc4k6hx'],\n", + " ['Bahamas',\n", + " 'location.country.languages_spoken',\n", + " 'Bahamas Creole English Language'],\n", + " ['m.064sml9', 'base.popstra.vacation_choice.vacationer', 'Cameron Diaz'],\n", + " ['LaToy Williams', 'common.topic.notable_for', 'g.1yl5s7m85'],\n", + " ['West Virginia',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'North Carolina'],\n", + " ['Cuba', 'meteorology.cyclone_affected_area.cyclones', 'Tropical Storm Fay'],\n", + " ['East Coast of the United States',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Hurricane Hazel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Eastern Canada'],\n", + " ['Jamie Lynn Spears',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Paradise Island', 'common.topic.notable_types', 'Geographical Feature'],\n", + " ['Cayman Islands', 'common.topic.notable_types', 'Country'],\n", + " ['Hurricane Edna',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1954 Atlantic hurricane season'],\n", + " ['Gregory Taylor', 'common.topic.article', 'm.0c01czp'],\n", + " ['Florida Keys', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['The Assemblies of God Bible College, Nassau, NP, Bahamas',\n", + " 'education.educational_institution.campuses',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Maine'],\n", + " ['South Andros Airport', 'location.location.containedby', 'Bahamas'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['m.0khm7ss', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['2000 Summer Olympics', 'olympics.olympic_games.sports', 'Sailing'],\n", + " ['Puerto Rico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mz2'],\n", + " ['Hurricane Tomas',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Central Abaco', 'location.location.containedby', 'Abaco Islands'],\n", + " ['m.0xntgpw', 'film.performance.actor', 'Fabienne Dali'],\n", + " ['Patrick Harris', 'common.topic.notable_types', 'Athlete'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4nfp5'],\n", + " ['District of the Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'City/Town/Village should pertain to Ghana, since City/Town/Village subdivides Ghanaian Municipal District.'],\n", + " ['Florida',\n", + " 'location.location.events',\n", + " 'Spain in the American Revolutionary War'],\n", + " ['Dominican Republic',\n", + " 'base.locations.countries.continent',\n", + " 'North America'],\n", + " ['Place of death', 'type.property.schema', 'Deceased Person'],\n", + " ['Bahamas', 'location.location.contains', 'Atlantis Royal Tower East'],\n", + " ['m.0khm64y',\n", + " 'olympics.olympic_athlete_affiliation.olympics',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Progressive Liberal Party'],\n", + " ['1949 Florida hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['Queen Victoria Platt', 'people.person.nationality', 'Bahamas'],\n", + " ['The Bluff, Bahamas', 'location.location.geolocation', 'm.0d5s81d'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['m.0796snd', 'olympics.olympic_medal_honor.medal', 'Bronze medal'],\n", + " ['2010 Commonwealth Games',\n", + " 'sports.multi_event_tournament.participating_countries',\n", + " 'Cayman Islands'],\n", + " ['Boxing',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '1972 Summer Olympics'],\n", + " ['1932 Bahamas hurricane',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1932 Atlantic hurricane season'],\n", + " ['Cuba', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Lili'],\n", + " ['Andros Town', 'common.topic.notable_types', 'Location'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4lc3f'],\n", + " ['United States of America',\n", + " 'location.location.primarily_containedby',\n", + " 'North America'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblns'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3fskt'],\n", + " ['1924 Cuba hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Central America'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc39kmc'],\n", + " ['Nick Cannon', 'people.person.ethnicity', 'African American'],\n", + " ['Dominican Republic',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1899 San Ciriaco hurricane'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblnj'],\n", + " ['Archipelago of the Azores',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gert'],\n", + " ['Harbour Island, Bahamas',\n", + " 'location.location.time_zones',\n", + " 'Eastern Time Zone'],\n", + " ['Hurricane Isaac',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['Mexico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['Hurricane Cleo', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc3l9dq'],\n", + " ['Honduras',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1948 Miami hurricane'],\n", + " ['Hurricane Ophelia',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '2005 Atlantic hurricane season'],\n", + " ['m.0khm64y',\n", + " 'olympics.olympic_athlete_affiliation.athlete',\n", + " 'Bianca Stuart'],\n", + " ['Haiti', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane David'],\n", + " ['1952 Summer Olympics', 'olympics.olympic_games.sports', 'Swimming'],\n", + " ['m.09w3hrf',\n", + " 'common.webpage.resource',\n", + " \"Exclusive: 'Ugly Betty' hits the beach!\"],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Atlantic Canada'],\n", + " ['Jonathan Barry', 'common.topic.article', 'm.0b__bbl'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Inez'],\n", + " ['Deceased Person', 'freebase.type_hints.included_types', 'Person'],\n", + " ['Americas', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['m.0nfblnj',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Sapodilla Creek', 'location.location.geolocation', 'm.0cn0ck4'],\n", + " ['Hurricane Inez', 'meteorology.tropical_cyclone.affected_areas', 'Haiti'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6f6y_'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4tjyd'],\n", + " ['Bahamas', 'location.location.contains', 'Nassau'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.12tb6gh26'],\n", + " ['Johnny Depp', 'people.person.nationality', 'United States of America'],\n", + " ['Massachusetts',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Noel'],\n", + " ['Allan Murray', 'people.person.gender', 'Male'],\n", + " ['m.0gczybd', 'film.performance.actor', 'Fabienne Dali'],\n", + " ['United Nations', 'organization.organization.founders', 'Honduras'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_xk19'],\n", + " ['Military Person', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Tourist attraction', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Patrick Harris',\n", + " 'sports.pro_athlete.sports_played_professionally',\n", + " 'm.0_gckh0'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'South Carolina'],\n", + " ['College of The Bahamas',\n", + " 'internet.top_level_domain_sponsor.domains',\n", + " 'bs'],\n", + " ['Joel Stubbs', 'common.topic.article', 'm.04mzrx3'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3sjbg'],\n", + " ['Swimming',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '1968 Summer Olympics'],\n", + " ['Political party', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mwn'],\n", + " ['m.0j1bz6l', 'common.webpage.topic', 'Bahamas'],\n", + " ['Mario Ford', 'common.topic.notable_for', 'g.12567r7rx'],\n", + " ['Bahamas', 'location.location.contains', \"Arthur's Town Airport\"],\n", + " ['Tropical Storm Fay', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['San Salvador and Rum Cay', 'location.location.geolocation', 'm.02_q_hz'],\n", + " ['Alice Town',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area code 242'],\n", + " ['Jamaica', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Noel'],\n", + " ['Acklins Island', 'common.topic.article', 'm.03xv2v'],\n", + " ['Canada',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['District of the Bahamas',\n", + " 'base.aareas.schema.administrative_area_type.sovereignty',\n", + " 'Bahamas'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Klaus'],\n", + " ['Hurricane Isabel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'New York'],\n", + " ['1932 Bahamas hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " [\"V'alonee Robinson\", 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4jpth'],\n", + " ['Sloan Farrington', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Basketball Player',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Athlete'],\n", + " ['m.0nf84h9',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Hurricane Ike', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Sandy Point', 'location.administrative_division.country', 'Bahamas'],\n", + " ['Place of death', 'rdf-schema#range', 'Location'],\n", + " ['Ramon Miller', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Jamaica', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Greta'],\n", + " ['Ragged Island, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['Sapodilla Creek', 'geography.river.basin_countries', 'Bahamas'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Klaus'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09wsn92'],\n", + " ['Democratic National Alliance',\n", + " 'common.topic.notable_types',\n", + " 'Organization'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc51dgt'],\n", + " ['Hurricane Floyd', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09yqq63'],\n", + " ['Freeport', 'location.location.people_born_here', 'Demetrius Pinder'],\n", + " ['1941 Florida hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['m.0khm15p', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Rum Cay District'],\n", + " [\"Dean's Blue Hole\", 'location.location.containedby', 'Bahamas'],\n", + " ['The Bluff, Bahamas', 'common.topic.article', 'm.0g43yg'],\n", + " ['Barbados', 'royalty.kingdom.rulers', 'Elizabeth II'],\n", + " ['Hurricane Katrina', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Cat Island, Bahamas', 'common.topic.image', 'Cat_island'],\n", + " ['Hurricane Katrina',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1981 Atlantic hurricane season'],\n", + " ['Marsh Harbour', 'common.topic.article', 'm.093p3_'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Coalition for Democratic Reform'],\n", + " ['Hurricane Noel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'North Carolina'],\n", + " ['m.0kd9w_5',\n", + " 'olympics.olympic_athlete_affiliation.olympics',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['m.04g8ppw',\n", + " 'government.government_position_held.office_position_or_title',\n", + " 'Queen of the Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc3qg5c'],\n", + " ['Freeport', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4_k3h'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Sloan Farrington'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mww'],\n", + " ['Jermaine Adderley', 'common.topic.article', 'm.0k6g8hf'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'West Virginia'],\n", + " ['m.06dn7md',\n", + " 'base.athletics.athletics_championships_competition_athlete_relationship.event',\n", + " \"2007 World Championships in Athletics – Men's 400 metres\"],\n", + " ['Freetown, Bahamas', 'common.topic.notable_for', 'g.1259_1v31'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.foreign_direct_investment_net_inflows',\n", + " 'g.1hhc4j_wd'],\n", + " ['Jermaine Adderley', 'people.person.profession', 'Cricketer'],\n", + " ['2004 Summer Olympics', 'olympics.olympic_games.sports', 'Swimming'],\n", + " ['Atlantic Canada', 'location.location.contains', 'Newfoundland'],\n", + " ['m.0nf6n2h',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_hints.included_types',\n", + " 'Administrative Area'],\n", + " ['Treasure Cay Airport', 'common.topic.article', 'm.02q17j8'],\n", + " ['1928 Okeechobee hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Carolina'],\n", + " ['Bahamas Taxi Cab Union', 'common.topic.article', 'm.08nmdr'],\n", + " ['Virginia',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ophelia'],\n", + " ['m.0nf8xk7',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Eastern Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1900 Galveston hurricane'],\n", + " ['South Andros', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['m.0j5n49m',\n", + " 'government.government_position_held.basic_title',\n", + " 'Prime minister'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0kd9w_5'],\n", + " ['m.0nf8xj_',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245zj913'],\n", + " ['Hurricane Isaac',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '2012 Atlantic hurricane season'],\n", + " ['m.0j7r7w8', 'sports.sports_team_roster.player', 'Jermaine Adderley'],\n", + " ['Newfoundland',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Bill'],\n", + " ['British Virgin Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Hugo'],\n", + " ['Gavin Christie', 'common.topic.notable_for', 'g.125658l4x'],\n", + " ['Hurricane Frances',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['m.04m0l7j', 'common.webpage.topic', 'Bahamian dollar'],\n", + " ['Tropical Storm Gilda',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Jamaica'],\n", + " ['1928 Okeechobee hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['Coopers Town',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area code 242'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.12cp_h_jb'],\n", + " ['Eastern Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1949 Florida hurricane'],\n", + " [\"John Travolta's son dies in the Bahamas\",\n", + " 'common.resource.annotations',\n", + " 'm.09wsn92'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4dy6r'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pd9q'],\n", + " ['Georgia', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Gregory Taylor', 'cricket.cricket_bowler.bowling_style', 'Left-handed'],\n", + " ['Sheniqua Ferguson', 'people.person.place_of_birth', 'Nassau'],\n", + " ['1932 Cuba hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Jamaica'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2004 Summer Olympics'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Chris'],\n", + " ['Patrick Harris', 'people.person.nationality', 'Bahamas'],\n", + " ['Tropical Storm Gilda', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " [\"Walker's Cay\", 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc40r9_'],\n", + " ['Cyril Stevenson', 'common.topic.notable_for', 'g.1yl5s17w4'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'm.0nf5f1m'],\n", + " ['Ron Butler', 'people.person.profession', 'Actor'],\n", + " ['Gilbert NMO Morris', 'common.topic.notable_types', 'Person'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245_kjqp'],\n", + " ['1938 New England hurricane',\n", + " 'common.topic.notable_types',\n", + " 'Tropical Cyclone'],\n", + " ['m.0nfblr7',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Bahamas Hotel, Catering and Allied Workers Union',\n", + " 'common.topic.article',\n", + " 'm.08nm7w'],\n", + " ['Hurricane Flora',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Jamaica'],\n", + " ['Spring Point Township', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Military Person', 'freebase.type_profile.strict_included_types', 'Person'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'West Virginia'],\n", + " ['Bahamas Ministry of Foreign Affairs',\n", + " 'common.topic.notable_for',\n", + " 'g.125556nvw'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Fox'],\n", + " ['James Eugene Brown', 'freebase.valuenotation.has_value', 'Place of death'],\n", + " ['Ragged Island, Bahamas', 'common.topic.article', 'm.03xr2g'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4nnfx'],\n", + " ['Bermuda',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Franklin'],\n", + " ['Boxer', 'freebase.type_profile.strict_included_types', 'Athlete'],\n", + " ['Amara Jones', 'olympics.olympic_athlete.country', 'm.0khlsrg'],\n", + " ['Black Point, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['North Abaco', 'common.topic.notable_for', 'g.125c1d6z3'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtg'],\n", + " ['Hurricane Emily',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1987 Atlantic hurricane season'],\n", + " ['Avard Moncur', 'people.person.profession', 'Track and field athlete'],\n", + " ['High Rock',\n", + " 'fictional_universe.fictional_setting.contained_by',\n", + " 'Races of The Elder Scrolls'],\n", + " ['m.010gg000',\n", + " 'government.government_position_held.basic_title',\n", + " 'Prime minister'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.brain_drain_percent',\n", + " 'g.1hhc4t_s6'],\n", + " ['Bahamas', 'location.location.contains', 'Coopers Town'],\n", + " ['Jamie Lynn Spears', 'people.person.gender', 'Female'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n3c'],\n", + " ['Whitcliff Atkinson', 'cricket.cricket_bowler.pace', 'Medium'],\n", + " ['Alabama',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Crooked Island, Bahamas', 'common.topic.article', 'm.03y6cz'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Virginia'],\n", + " ['Gregory Taylor', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Hurricane Edna', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Marvin Rolle'],\n", + " ['m.07w429x',\n", + " 'olympics.olympic_medal_honor.olympics',\n", + " '2008 Summer Olympics'],\n", + " ['Hurricane Isabel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Puerto Rico'],\n", + " ['Inagua', 'location.location.geolocation', 'm.02_r6r7'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Eastern Canada'],\n", + " ['Hurricane Greta',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Southeastern United States'],\n", + " ['Roxie Roker', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Perry Christie',\n", + " 'government.politician.government_positions_held',\n", + " 'm.0n3k846'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3tknp'],\n", + " ['Americas', 'location.location.time_zones', 'Central Time Zone'],\n", + " ['Greater Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Sandy'],\n", + " ['Arthur Loves Plastic', 'people.person.gender', 'Female'],\n", + " ['m.0l3b605',\n", + " 'olympics.olympic_medal_honor.olympics',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['United Nations', 'organization.organization.founders', 'Canada'],\n", + " ['East Coast of the United States',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Floyd'],\n", + " ['Tom Robinson', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Bahamas Taxi Cab Union',\n", + " 'organization.organization.organization_type',\n", + " 'Trade union'],\n", + " ['Spanish Wells', 'location.statistical_region.population', 'g.11bc87_hq0'],\n", + " ['Tropical Storm Mindy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Cuba', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Tomas'],\n", + " ['Bahamas', 'location.country.first_level_divisions', \"Moore's Island\"],\n", + " ['Hurricane Hugo', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['m.0j5n49x',\n", + " 'government.government_position_held.basic_title',\n", + " 'Governor-General'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc4s847'],\n", + " ['Shamar Sands', 'olympics.olympic_athlete.country', 'm.0khm3h0'],\n", + " ['Black Point, Bahamas', 'common.topic.article', 'm.0c65v7'],\n", + " ['Americas', 'location.location.contains', 'Bermuda'],\n", + " ['Turks and Caicos Islands', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['Map of Bahamas', 'common.image.appears_in_topic_gallery', 'Inagua'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Cat Island, Bahamas', 'location.location.contains', 'New Bight Airport'],\n", + " ['Sailing',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '1972 Summer Olympics'],\n", + " ['Bahamas', 'location.location.contains', 'Tarpum Bay'],\n", + " ['Kate Bosworth', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Sailing',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '2000 Summer Olympics'],\n", + " ['Hurricane David',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Georgia'],\n", + " ['Bahamas', 'location.statistical_region.net_migration', 'g.1s067rs_9'],\n", + " ['Christine Amertil',\n", + " 'sports.tournament_event_competitor.events_competed_in',\n", + " 'm.0b64ynk'],\n", + " ['Atlantic Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Diana'],\n", + " ['m.0nf84lc',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Ragged Island, Bahamas',\n", + " 'common.topic.notable_types',\n", + " 'Administrative Division'],\n", + " ['Barbados',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Edith'],\n", + " ['Atlantic Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ophelia'],\n", + " ['1949 Florida hurricane',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1949 Atlantic hurricane season'],\n", + " ['m.0nf8xdm',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " [\"Governor's Harbour Airport\", 'location.location.geolocation', 'm.02_fjxv'],\n", + " ['Atlantis Royal Tower East', 'common.topic.notable_for', 'g.12pkcmckv'],\n", + " [\"Governor's Harbour Airport\",\n", + " 'aviation.airport.serves',\n", + " \"Governor's Harbour\"],\n", + " ['Hurricane Rita',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida Panhandle'],\n", + " ['Tureano Johnson', 'people.person.gender', 'Male'],\n", + " ['m.0lr0zpz',\n", + " 'government.political_party_tenure.party',\n", + " 'Progressive Liberal Party'],\n", + " ['Shamar Sands', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Dunmore Town',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area code 242'],\n", + " ['Parliamentary system',\n", + " 'government.form_of_government.countries',\n", + " 'Jamaica'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n01'],\n", + " ['Puerto Rico',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Chris'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Fabienne Dali'],\n", + " ['Alabama', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Erin'],\n", + " ['Bahamas',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Free National Movement'],\n", + " ['Allan Murray', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Tropical Storm Fay',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Louisiana'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1926 Nassau hurricane'],\n", + " ['Andrae Williams', 'people.person.nationality', 'Bahamas'],\n", + " ['Andros Town International Airport',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khly48'],\n", + " ['Lucayan Archipelago',\n", + " 'location.location.contains',\n", + " 'Turks and Caicos Islands'],\n", + " ['Hurricane Earl',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Eastern United States'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm15p'],\n", + " ['Ramon Miller', 'olympics.olympic_athlete.country', 'm.0khly48'],\n", + " ['Georgia',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Frances'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3m_rc'],\n", + " ['Michael Mathieu', 'people.person.profession', 'Track and field athlete'],\n", + " ['South Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ophelia'],\n", + " ['Cuba',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['Trevor Barry', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Albert Roker', 'common.topic.notable_types', 'Person'],\n", + " ['Shamar Sands', 'people.person.profession', 'Track and field athlete'],\n", + " ['Anguilla', 'location.country.official_language', 'English Language'],\n", + " ['Louisiana',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Rita'],\n", + " ['Long Island', 'location.location.containedby', 'Bahamas'],\n", + " ['Hope Town', 'location.location.containedby', 'Abaco Islands'],\n", + " ['1928 Okeechobee hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Georgia'],\n", + " ['Rock Sound International Airport',\n", + " 'location.location.containedby',\n", + " 'Bahamas'],\n", + " ['m.0b6vjr_', 'common.webpage.topic', 'Bahamas Airline Pilots Association'],\n", + " ['Robert Eardley', 'people.person.gender', 'Male'],\n", + " ['Dwight Weakley', 'cricket.cricket_bowler.bowling_style', 'Left-handed'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc41w_t'],\n", + " ['Nicaragua',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'United Nations'],\n", + " ['Georgia',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1928 Okeechobee hurricane'],\n", + " ['Hurricane Gordon',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['Inagua', 'location.statistical_region.population', 'g.11bymn4lr9'],\n", + " ['Bianca Stuart', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Britney Spears', 'people.person.profession', 'Singer'],\n", + " ['Hurricane Ike', 'meteorology.tropical_cyclone.affected_areas', 'Texas'],\n", + " ['m.0nfblq1',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xg_'],\n", + " ['Honduras',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Isbell'],\n", + " ['Johnny Depp', 'base.popstra.celebrity.vacations_in', 'm.065pc4_'],\n", + " ['Whitcliff Atkinson', 'common.topic.notable_for', 'g.1256d4w28'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Pauline Davis-Thompson'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4s853'],\n", + " ['Marsh Harbour', 'location.location.geolocation', 'm.02_kbvc'],\n", + " ['Hurricane Cleo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Atlantic Canada'],\n", + " ['1949 Florida hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Cuba'],\n", + " ['m.0khly48',\n", + " 'olympics.olympic_athlete_affiliation.olympics',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['South Abaco', 'location.location.geolocation', 'm.0wmqzh9'],\n", + " ['Reginald James Poitier', 'people.person.profession', 'Farmer'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbls_'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Ohio'],\n", + " ['Estelle Evans', 'people.person.gender', 'Female'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khm1yz'],\n", + " ['Europe',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ophelia'],\n", + " ['Coalition for Democratic Reform',\n", + " 'common.topic.notable_for',\n", + " 'g.1259f0m86'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Hubert Minnis'],\n", + " ['Kelly Clarkson',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0nktnws', 'music.group_membership.member', 'Eric Gibson'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc47bf9'],\n", + " ['Hurricane Betsy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['m.0nf6n15',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " [\"Abraham's Bay\", 'location.statistical_region.population', 'g.11bymm3d2t'],\n", + " ['Bahamas', 'location.location.contains', 'Snug Corner'],\n", + " ['Dominican Republic',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Sovereign state'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc449nk'],\n", + " ['Uma Thurman', 'base.popstra.celebrity.vacations_in', 'm.064ssr7'],\n", + " ['Laverne Eve', 'people.person.nationality', 'Bahamas'],\n", + " ['Patrick Harris', 'sports.pro_athlete.teams', 'm.0_gck1d'],\n", + " ['Florida', 'location.location.time_zones', 'Central Time Zone'],\n", + " ['Grand Bahama', 'location.location.people_born_here', 'Hubert Ingraham'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xj_'],\n", + " ['Laverne Eve', 'people.person.gender', 'Female'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Arthur Foulkes', 'people.person.place_of_birth', 'Matthew Town'],\n", + " ['m.09x2k1q', 'common.webpage.topic', 'Bahamas'],\n", + " ['Hurricane Edith',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Barbados'],\n", + " ['Nicaragua',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Michelle'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc40jvz'],\n", + " ['m.0khlf_8', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['m.064sml9', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Belize',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'City/Town/Village should pertain to Ghana, since City/Town/Village subdivides Ghanaian Municipal District.'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3pym2'],\n", + " ['Sailing',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['m.0nf6mvb',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Parliament of the Bahamas',\n", + " 'government.governmental_body.component_bodies',\n", + " 'Senate of the Bahamas'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc4gqnx'],\n", + " ['North America', 'base.locations.continents.countries_within', 'Nicaragua'],\n", + " ['United States Virgin Islands',\n", + " 'location.location.containedby',\n", + " 'North America'],\n", + " ['Belize', 'location.location.time_zones', 'Central Time Zone'],\n", + " ['Shonel Ferguson', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Maine', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Floyd'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblps'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc53l3f'],\n", + " ['Gale of 1878',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Pennsylvania'],\n", + " ['Gilbert NMO Morris', 'people.person.gender', 'Male'],\n", + " ['Jamaica', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Erin'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4mtxm'],\n", + " ['Hurricane Ophelia',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'North Carolina'],\n", + " ['Flag of the Bahamas', 'symbols.flag.used_by', 'm.0jqg_bs'],\n", + " ['m.0nf84k1',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Cayman Islands',\n", + " 'location.country.form_of_government',\n", + " 'Parliamentary system'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245zkdm9'],\n", + " [\"V'alonee Robinson\", 'people.person.profession', 'Track and field athlete'],\n", + " ['Stafford Sands', 'common.topic.notable_types', 'Deceased Person'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_q10k'],\n", + " ['South Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gordon'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Central United States'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Dennis'],\n", + " ['Ohio',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['Bahamas', 'location.location.contains', 'Exuma'],\n", + " ['Rum Cay District',\n", + " 'location.location.nearby_airports',\n", + " 'Port Nelson Airport'],\n", + " ['Progressive Liberal Party',\n", + " 'government.political_party.ideology',\n", + " 'Social liberalism'],\n", + " ['Wesley Neymour', 'olympics.olympic_athlete.country', 'm.0khl_xg'],\n", + " ['Texas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Rita'],\n", + " ['Government Agency', 'freebase.type_hints.included_types', 'Organization'],\n", + " ['Hurricane Greta',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Lesser Antilles'],\n", + " ['Cat Island, Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['1919 Florida Keys hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Texas'],\n", + " ['Godfrey Ellis', 'people.person.education', 'm.0772hrw'],\n", + " ['Reginald James Poitier', 'people.person.children', 'Sidney Poitier'],\n", + " ['1935 Labor Day hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida Panhandle'],\n", + " ['Harbour Island', 'common.topic.notable_types', 'Administrative Division'],\n", + " ['Hurricane Edna', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['1947 Fort Lauderdale hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['2009 World Championships in Athletics',\n", + " 'sports.multi_event_tournament.sports',\n", + " 'Track and field athletics'],\n", + " ['Green Turtle Cay District',\n", + " 'location.location.containedby',\n", + " 'Green Turtle Cay'],\n", + " ['Derrick Atkins', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Tureano Johnson', 'martial_arts.martial_artist.martial_art', 'Boxing'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'New York'],\n", + " ['Jamaica', 'location.location.containedby', 'Americas'],\n", + " ['Bermuda',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Tropical Storm Leslie'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'Central Andros'],\n", + " ['Mexico',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['m.010rj0_n', 'film.performance.actor', 'Fabienne Dali'],\n", + " ['Bahamian dollar', 'common.topic.article', 'm.01l6dv'],\n", + " ['The Cove Atlantis', 'location.location.containedby', 'Paradise Island'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mtp'],\n", + " ['Canada', 'royalty.kingdom.rulers', 'Elizabeth II'],\n", + " ['Exuma', 'location.location.people_born_here', 'Joan Sawyer'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Whitcliff Atkinson', 'people.person.gender', 'Male'],\n", + " ['Building', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Sandy'],\n", + " ['Puerto Rico',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1891 Martinique hurricane'],\n", + " ['Mississippi', 'location.location.containedby', 'United States of America'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc403l0'],\n", + " ['Geographical Feature', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Donald Knowles', 'common.topic.article', 'm.063ymdr'],\n", + " ['Alabama', 'location.location.containedby', 'United States of America'],\n", + " ['m.04g8ppw',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['Maine',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1938 New England hurricane'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4vmc8'],\n", + " ['New Bight Airport', 'common.topic.notable_types', 'Airport'],\n", + " ['Blake Bartlett', 'common.topic.notable_types', 'Athlete'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['San Salvador Island', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.065pc4_'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Mississippi'],\n", + " ['Political party', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Maryland', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['Basketball Player', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3z8cs'],\n", + " ['Rum Cay District',\n", + " 'common.topic.notable_types',\n", + " 'Administrative Division'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.category',\n", + " 'Category 3 Hurricane (SSHS)'],\n", + " ['Cat Island, Bahamas', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Hurricane Hanna',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Turks and Caicos Islands'],\n", + " ['Amara Jones', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Henry Milton Taylor', 'people.person.gender', 'Male'],\n", + " ['Leeward Islands Airline Pilots Association',\n", + " 'common.topic.notable_types',\n", + " 'Organization'],\n", + " ['Hubert Ingraham',\n", + " 'government.politician.government_positions_held',\n", + " 'm.0j5n49m'],\n", + " ['Hubert Ingraham', 'people.person.profession', 'Politician'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblpj'],\n", + " ['Unitary state', 'government.form_of_government.countries', 'Nicaragua'],\n", + " ['Hurricane Noel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'East Coast of the United States'],\n", + " ['Oneil Levy', 'common.topic.notable_types', 'Cricket Player'],\n", + " ['United States Virgin Islands',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " [\"Deadman's Cay Airport\", 'common.topic.notable_types', 'Airport'],\n", + " ['Bahamas national football team', 'sports.sports_team.roster', 'm.0j3wyf_'],\n", + " ['South Florida',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Andrew'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Amara Jones'],\n", + " ['Florida',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Leevan Sands', 'people.person.gender', 'Male'],\n", + " ['Bahamas', 'location.location.contains', 'Spring Point Airport'],\n", + " ['Hurricane Jeanne',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '2004 Atlantic hurricane season'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3f_cn'],\n", + " ['Eric Gibson', 'common.topic.article', 'm.0876bm'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Bermuda'],\n", + " ['Hurricane Carol', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Marsh Harbour', 'location.location.containedby', 'Bahamas'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Turks and Caicos Islands'],\n", + " ['Hurricane Isbell',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Eastern Canada'],\n", + " ['Bahamas',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'Parliament of the Bahamas'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Location'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Albert Roker'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6flvm'],\n", + " ['Shaunae Miller', 'people.person.nationality', 'Bahamas'],\n", + " ['Gale of 1878',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'New England'],\n", + " ['Sloan Farrington', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Cuba', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Katrina'],\n", + " ['Pennsylvania',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Llewellyn Armstrong'],\n", + " ['1948 Miami hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Honduras'],\n", + " ['m.0vskksj', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Bahamas', 'sports.sport_country.athletic_performances', 'm.0b64vms'],\n", + " ['m.04p8x9v',\n", + " 'organization.organization_membership.organization',\n", + " 'World Bank'],\n", + " ['Barbados', 'common.topic.notable_types', 'Country'],\n", + " ['Marvin Rolle', 'common.topic.notable_types', 'Tennis Player'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6myn'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc52vkr'],\n", + " ['Jamaica',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1245_mywj'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4dlq2'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.sports',\n", + " 'Track and field athletics'],\n", + " ['Stella Maris Airport', 'common.topic.article', 'm.0bs620p'],\n", + " ['Bahamas Creole English Language',\n", + " 'common.topic.notable_types',\n", + " 'Human Language'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4f7vc'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.energy_use_per_capita',\n", + " 'g.1245_d2kc'],\n", + " ['Mount Creek', 'location.location.geolocation', 'm.0cw_xnf'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['1935 Labor Day hurricane',\n", + " 'meteorology.tropical_cyclone.tropical_cyclone_season',\n", + " '1935 Atlantic hurricane season'],\n", + " ['Louisiana',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Florida'],\n", + " ['1935 Labor Day hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Carolina'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3h35y'],\n", + " ['Treasure Cay',\n", + " 'location.location.nearby_airports',\n", + " 'Treasure Cay Airport'],\n", + " ['Avard Moncur', 'olympics.olympic_athlete.country', 'm.0khl_62'],\n", + " ['Jamaica',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gordon'],\n", + " ['Hurricane Wilma', 'meteorology.tropical_cyclone.affected_areas', 'Belize'],\n", + " ['m.0khm88z', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Dominican Republic',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Noel'],\n", + " ['Starve Creek', 'common.topic.notable_for', 'g.1259nbx65'],\n", + " ['Nichollstown and Berry Islands', 'common.topic.article', 'm.03y1r5'],\n", + " ['m.0nf8xbm',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Massachusetts'],\n", + " ['Eastern Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ike'],\n", + " ['Mayaguana', 'location.administrative_division.country', 'Bahamas'],\n", + " ['North Eleuthera', 'common.topic.notable_types', 'Administrative Division'],\n", + " ['m.063g0cs', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['North Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Noel'],\n", + " ['Shonel Ferguson', 'common.topic.article', 'm.04zyqgs'],\n", + " ['Gary Armstrong', 'common.topic.notable_types', 'Cricket Player'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'g.1q5jkdsnf'],\n", + " ['1947 Fort Lauderdale hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['New York', 'location.location.containedby', 'United States of America'],\n", + " ['Chelsea Handler', 'people.person.nationality', 'United States of America'],\n", + " ['North Carolina',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Floyd'],\n", + " ['Bahamas',\n", + " 'location.location.contains',\n", + " 'The Assemblies of God Bible College, Nassau, NP, Bahamas'],\n", + " ['North Andros', 'common.topic.notable_types', 'Administrative Division'],\n", + " ['Football player', 'freebase.type_profile.strict_included_types', 'Person'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4bpnc'],\n", + " ['Haiti', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['Sailing',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '1968 Summer Olympics'],\n", + " ['Central America', 'location.location.contains', 'Belize'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Irene'],\n", + " ['Olympic athlete', 'freebase.type_profile.strict_included_types', 'Person'],\n", + " ['Donald Thomas', 'people.person.place_of_birth', 'Freeport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc4spw5'],\n", + " ['Deceased Person', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0nfblqy',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Georgia',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Dennis'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Katrina'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3m7hf'],\n", + " ['Lindsay Lohan', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Parliamentary system',\n", + " 'government.form_of_government.countries',\n", + " 'Cayman Islands'],\n", + " ['Robert Eardley', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Congo Town', 'common.topic.article', 'm.064n6t4'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n08'],\n", + " ['Harbour Island District',\n", + " 'common.topic.notable_types',\n", + " 'Administrative Division'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Rico Forbes'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3f_jp'],\n", + " ['Eastern Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Isbell'],\n", + " ['2009 World Championships in Athletics',\n", + " 'sports.multi_event_tournament.competitions',\n", + " \"2009 World Championships in Athletics – Women's 4 × 100 metres relay\"],\n", + " ['Cricketer', 'freebase.equivalent_topic.equivalent_type', 'Cricket Player'],\n", + " ['Exuma', 'location.location.containedby', 'Bahamas'],\n", + " ['Ulrich Fox', 'people.person.nationality', 'Canada'],\n", + " ['Venris Bennett', 'common.topic.notable_types', 'Cricket Player'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3m3rg'],\n", + " ['Bahamas', 'base.popstra.location.vacationers', 'm.064cvnf'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Reginald James Poitier'],\n", + " ['Jackner Louis', 'people.person.gender', 'Male'],\n", + " ['Nicaragua',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3b9n6'],\n", + " ['Bahamas', 'location.location.contains', 'Clarence Town'],\n", + " ['Rum Cay District', 'common.topic.notable_for', 'g.1259dwrv9'],\n", + " ['Bermuda', 'location.country.official_language', 'English Language'],\n", + " ['District of the Bahamas', 'type.type.domain', 'Administrative Areas'],\n", + " ['Mangrove Cay',\n", + " 'location.location.nearby_airports',\n", + " 'Clarence A. Bain Airport'],\n", + " ['Dominica',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2008 Summer Olympics'],\n", + " ['Bahamas', 'location.country.first_level_divisions', 'West Grand Bahama'],\n", + " ['m.0nf5f15',\n", + " 'measurement_unit.dated_kgoe.source',\n", + " 'Energy use per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Map of Bahamas',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'Andros, Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4_wbb'],\n", + " ['2007 World Championships in Athletics',\n", + " 'sports.multi_event_tournament.sports',\n", + " 'Track and field athletics'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc3kp4n'],\n", + " ['Dwight Weakley', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Anthonique Strachan', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Whitcliff Atkinson', 'people.person.profession', 'Cricketer'],\n", + " ['Ulrich Fox', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'olympics.olympic_participating_country.medals_won',\n", + " 'm.0l3b605'],\n", + " ['Hurricane Ike', 'meteorology.tropical_cyclone.affected_areas', 'Haiti'],\n", + " ['Jimmy Spice Curry', 'people.person.gender', 'Male'],\n", + " ['Cuba',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2008 Summer Olympics'],\n", + " ['Abaco Islands', 'location.location.contains', 'North Abaco'],\n", + " ['Roxie Roker', 'people.person.parents', 'Albert Roker'],\n", + " ['Hurricane Ike',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Louisiana'],\n", + " ['Starve Creek', 'common.topic.article', 'm.06w8vqc'],\n", + " ['Bennet Davis', 'common.topic.notable_for', 'g.1257xjffl'],\n", + " ['Hurricane Flora', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1926 Nassau hurricane'],\n", + " ['Queen Victoria Platt', 'common.topic.notable_types', 'Person'],\n", + " ['Hurricane Bill', 'meteorology.tropical_cyclone.affected_areas', 'Europe'],\n", + " ['Bahamas', 'location.country.official_language', 'English Language'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Hazel'],\n", + " ['Hurricane Isaac',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Louisiana'],\n", + " ['Bahamas', 'olympics.olympic_participating_country.athletes', 'm.0khl4t6'],\n", + " ['Bahamas',\n", + " 'olympics.olympic_participating_country.medals_won',\n", + " 'm.0796snd'],\n", + " ['1945 Homestead hurricane',\n", + " 'common.topic.notable_types',\n", + " 'Tropical Cyclone'],\n", + " [\"V'alonee Robinson\", 'people.person.place_of_birth', 'New Providence'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.deposit_interest_rate',\n", + " 'g.1hhc3mbqz'],\n", + " ['Dereck Gittens', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Snug Corner', 'common.topic.article', 'm.0g43ql'],\n", + " ['Debbie Ferguson-McKenzie', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc3gxmy'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3h6ts'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Isabel'],\n", + " ['Perry Christie', 'people.person.profession', 'Politician'],\n", + " ['Olympic athlete', 'freebase.type_hints.included_types', 'Athlete'],\n", + " ['Treasure Cay', 'common.topic.notable_for', 'g.1255z33mv'],\n", + " ['Michael Mathieu', 'people.person.place_of_birth', 'Freeport'],\n", + " ['Spring Point Township',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Hurricane Inez', 'meteorology.tropical_cyclone.affected_areas', 'Bahamas'],\n", + " ['Gregory Taylor', 'people.person.nationality', 'Bahamas'],\n", + " ['m.0nfblnj',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.12460_128'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'West Virginia'],\n", + " ['North Carolina',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Hurricane Noel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Turks and Caicos Islands'],\n", + " ['Cayman Islands',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['Guadeloupe',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1928 Okeechobee hurricane'],\n", + " ['East Coast of the United States',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Arthur'],\n", + " ['High Rock', 'location.location.containedby', 'Bahamas'],\n", + " ['East Coast of the United States',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Donna'],\n", + " ['Central Abaco', 'location.location.geolocation', 'm.0wmp81k'],\n", + " ['Hurricane Sandy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Greater Antilles'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mxj'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1q5jprczg'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245z0x2m'],\n", + " ['Earth', 'base.locations.planets.continents_within', 'Europe'],\n", + " ['Dominican Republic',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Edith'],\n", + " ['New York',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc474by'],\n", + " ['m.07w429x',\n", + " 'olympics.olympic_medal_honor.event',\n", + " \"Athletics at the 2008 Summer Olympics – Men's triple jump\"],\n", + " ['Jonathan Barry', 'cricket.cricket_player.batting_style', 'Right-handed'],\n", + " ['Shaunae Miller', 'olympics.olympic_athlete.country', 'm.0khlz57'],\n", + " ['m.07w429x', 'olympics.olympic_medal_honor.country', 'Bahamas'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'North Carolina'],\n", + " ['Swimming',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " '2004 Summer Olympics'],\n", + " ['Anguilla', 'common.topic.notable_types', 'Country'],\n", + " ['m.0645m6v', 'base.popstra.vacation_choice.vacationer', 'Amanda Bynes'],\n", + " ['Hurricane Michelle',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Lucaya, Bahamas', 'common.topic.notable_types', 'Location'],\n", + " ['Eric Gibson', 'people.person.gender', 'Male'],\n", + " ['Mexico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['Hurricane Edna', 'meteorology.tropical_cyclone.affected_areas', 'Maine'],\n", + " ['Earth',\n", + " 'base.locations.planets.countries_within',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Virginia'],\n", + " ['Sandy Point', 'common.topic.notable_types', 'Administrative Division'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3n_6p'],\n", + " ['Jamaica',\n", + " 'sports.sport_country.multi_event_tournaments_participated_in',\n", + " '2007 World Championships in Athletics'],\n", + " ['Sidney Poitier', 'people.person.nationality', 'United States of America'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Irene'],\n", + " ['m.064tk7l', 'base.popstra.vacation_choice.location', 'Bahamas'],\n", + " ['Greater Antilles', 'location.location.contains', 'Cayman Islands'],\n", + " ['Marvin Rolle', 'common.topic.article', 'm.0hgns2h'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245z6k03'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Andrew'],\n", + " ['Sovereign state',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Administrative Area'],\n", + " ['Bahamas',\n", + " 'location.country.form_of_government',\n", + " 'Constitutional monarchy'],\n", + " ['Bahamas', 'location.statistical_region.cpi_inflation_rate', 'g.1hhc41wwv'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc3pyrj'],\n", + " ['m.0khly48',\n", + " 'olympics.olympic_athlete_affiliation.athlete',\n", + " 'Ramon Miller'],\n", + " ['Andrae Williams', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Americas', 'location.location.contains', 'United States of America'],\n", + " ['Cayman Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Paloma'],\n", + " ['Progressive Liberal Party',\n", + " 'organization.organization.founders',\n", + " 'William Cartwright'],\n", + " ['Hurricane Paloma',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zcv02'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc3m_mj'],\n", + " ['m.0v_d21t', 'people.marriage.spouse', 'Ethel Lee'],\n", + " ['Hurricane Frances',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Turks and Caicos Islands'],\n", + " ['New England', 'location.location.containedby', 'Eastern United States'],\n", + " ['Hurricane Katrina',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Florida'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc3xpwv'],\n", + " ['New Jersey',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['Lindsay Lohan', 'people.person.profession', 'Actor'],\n", + " ['Tropical Storm Franklin',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['Johnny Depp', 'people.person.profession', 'Actor'],\n", + " ['Atlantic Canada',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Bill'],\n", + " ['West Virginia',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['District of the Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'Geneva should be contained in some Scottish district, German rural district, Beninese sub-prefectures, Afghan district, East Timorese subdistrict, Belizean constituency, Bangladeshi district, New Zealand district, Seychellois district, Burmese district, Parish of the Isle of Man, Lebanese county, Ghanaian metropolitan district, Thai district, South Korean county, District of the Philippines, Nigerian local government area, Congolese zone, Cape Verdian parish, Papua New Guinean district, Jordanian sub-district, District of Saint Helena, Omani district, Israeli subdistrict, Lithuanian district municipality, Canton of Luxembourg, County of Croatia, Dominican parish, Spanish comarca, Libyan district, District of the Bahamas, Cambodian district, District of the French Southern Territories, District of São Tomé and Príncipe, Paraguayan district, District of Mauritius, Bruneian mukim, Commune of the Faroe Islands, Swiss district, Ghanaian district, Austrian political district, South African district, District of Nauru, German urban district, Slovak district, Polish county, Brazilian municipality, Northern Irish district, Albanian bashkia, Central African sub-prefecture, Hungarian urban county, Canadian county, District of Sierra Leone, Congolese district, Belarusian district, Romanian county, Subdistrict of Botswana, Ethiopian zone, Tanzanian district, County of South Sudan, Belgian arrondissement, Irish county, Rwandan district, District of Hong Kong, Indian subdivision, District of Anguilla, Moldovan district, Surinamese ressort, Syrian district, Cueillete of Jersey, Liberian district, District of Serbia, Somalian district, Kenyan district, Ghanaian ordinary district, Nepalese district, District of Laos, Iranian county, Cayman Islands district, Vingtaine of Jersey, Quarter of Saint Lucia, Angolan council, Gambian district, Singapore district, Guinean subprefecture, Malaysian district, Taiwanese district, Panamanian district, District of Kyrgyzstan, Albanian komuna, Czech district, Indonesian subdistrict, Sudanese district, District of Mozambique, Samoan district, Argentinian department, District of Malawi, Local council of Malta, Ecuadorean canton, Costa Rican canton, or Ghanaian municipal district within Canton of Geneva.'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc489d9'],\n", + " ['Pauline Davis-Thompson', 'people.person.nationality', 'Bahamas'],\n", + " ['Hurricane Betsy',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Lesser Antilles'],\n", + " ['m.0nfblrh',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['m.07zj6w7', 'olympics.olympic_medal_honor.country', 'Bahamas'],\n", + " ['Abali Hoilett', 'people.person.gender', 'Male'],\n", + " ['Honduras',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Wilma'],\n", + " ['Cayman Islands',\n", + " 'location.country.form_of_government',\n", + " 'Constitutional monarchy'],\n", + " ['Honduras',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Gordon'],\n", + " ['Colonel Hill',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area code 242'],\n", + " ['Administrative Division',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Location'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1891 Martinique hurricane'],\n", + " ['West Virginia', 'location.location.containedby', 'Southern United States'],\n", + " ['1945 Homestead hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Bahamas'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.category',\n", + " 'Category 4 Hurricane (SSHS)'],\n", + " ['m.0n3k82w',\n", + " 'government.government_position_held.office_position_or_title',\n", + " 'Prime Minister of the Bahamas'],\n", + " ['m.0n3k846',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'Bahamas'],\n", + " ['m.0796snn',\n", + " 'olympics.olympic_medal_honor.olympics',\n", + " '2004 Summer Olympics'],\n", + " [\"Rosie O'Donnell\", 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Maryland',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['Turks and Caicos Islands',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1949 Florida hurricane'],\n", + " ['Prime Minister of the Bahamas',\n", + " 'government.government_office_or_title.office_holders',\n", + " 'm.0j5n49m'],\n", + " ['Bahamas', 'common.topic.notable_for', 'g.1255xf8w1'],\n", + " ['1900 Galveston hurricane',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'New York'],\n", + " ['District of the Bahamas',\n", + " 'base.uncommon.topic.exceptions',\n", + " 'Basel should be contained in some Scottish district, German rural district, Beninese sub-prefectures, Afghan district, East Timorese subdistrict, Belizean constituency, Bangladeshi district, New Zealand district, Seychellois district, Burmese district, Parish of the Isle of Man, Lebanese county, Ghanaian metropolitan district, Thai district, South Korean county, District of the Philippines, Nigerian local government area, Congolese zone, Cape Verdian parish, Papua New Guinean district, Jordanian sub-district, District of Saint Helena, Omani district, Israeli subdistrict, Lithuanian district municipality, Canton of Luxembourg, County of Croatia, Dominican parish, Spanish comarca, Libyan district, District of the Bahamas, Cambodian district, District of the French Southern Territories, District of São Tomé and Príncipe, Paraguayan district, District of Mauritius, Bruneian mukim, Commune of the Faroe Islands, Swiss district, Ghanaian district, Austrian political district, South African district, District of Nauru, German urban district, Slovak district, Polish county, Brazilian municipality, Northern Irish district, Albanian bashkia, Central African sub-prefecture, Hungarian urban county, Canadian county, District of Sierra Leone, Congolese district, Belarusian district, Romanian county, Subdistrict of Botswana, Ethiopian zone, Tanzanian district, County of South Sudan, Belgian arrondissement, Irish county, Rwandan district, District of Hong Kong, Indian subdivision, District of Anguilla, Moldovan district, Surinamese ressort, Syrian district, Cueillete of Jersey, Liberian district, District of Serbia, Somalian district, Kenyan district, Ghanaian ordinary district, Nepalese district, District of Laos, Iranian county, Cayman Islands district, Vingtaine of Jersey, Quarter of Saint Lucia, Angolan council, Gambian district, Singapore district, Guinean subprefecture, Malaysian district, Taiwanese district, Panamanian district, District of Kyrgyzstan, Albanian komuna, Czech district, Indonesian subdistrict, Sudanese district, District of Mozambique, Samoan district, Argentinian department, District of Malawi, Local council of Malta, Ecuadorean canton, Costa Rican canton, or Ghanaian municipal district within Basel-City.'],\n", + " ['Bahamas', 'location.location.people_born_here', 'Pleasant Bridgewater'],\n", + " ['Ohio', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Isabel'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6n28'],\n", + " ['Rico Forbes', 'people.person.parents', 'Eric Forbes'],\n", + " ['Sandy Creek (Bahamas)', 'common.topic.article', 'm.06wbgt6'],\n", + " ['Hurricane Gordon', 'meteorology.tropical_cyclone.affected_areas', 'Haiti'],\n", + " ['Bahamas',\n", + " 'base.athletics.athletics_country.championships_athletes_performances',\n", + " 'm.06dn7md'],\n", + " ['Anita Allen', 'common.topic.article', 'm.0k9j33q'],\n", + " ['Spanish Wells', 'location.location.containedby', 'Bahamas'],\n", + " ['Avard Moncur', 'people.person.place_of_birth', 'Nassau'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfblr_'],\n", + " ['Pennsylvania',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Bahamas', 'location.statistical_region.gni_in_ppp_dollars', 'm.0nfbll0'],\n", + " ['Mississippi',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1947 Fort Lauderdale hurricane'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Massachusetts'],\n", + " ['Central Eleuthera', 'common.topic.notable_for', 'g.12575mzpx'],\n", + " ['East Grand Bahama', 'location.location.geolocation', 'm.0wmtnd7'],\n", + " ['Coalition for Democratic Reform', 'common.topic.article', 'm.03f6sv'],\n", + " ['Ethel Lee', 'people.person.gender', 'Female'],\n", + " ['Canada', 'location.location.containedby', 'Americas'],\n", + " ['Americas', 'location.location.contains', 'Turks and Caicos Islands'],\n", + " ['Grand Bahama',\n", + " 'location.location.contains',\n", + " 'Grand Bahama International Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124607cj6'],\n", + " ['m.0l_g48p', 'people.sibling_relationship.sibling', 'Estelle Evans'],\n", + " ['Clinton Outten', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'British Virgin Islands'],\n", + " ['College of The Bahamas', 'location.location.containedby', 'Bahamas'],\n", + " ['m.0nf8xgr',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nf8xcm'],\n", + " ['Sovereign state', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0nfbls_',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.gdp_nominal_per_capita',\n", + " 'g.1hhc39ccd'],\n", + " ['Ivanique Kemp', 'common.topic.notable_types', 'Olympic athlete'],\n", + " ['Building', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc42rnn'],\n", + " ['Puerto Rico',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1926 Nassau hurricane'],\n", + " ['Cause of death', 'rdf-schema#domain', 'Deceased Person'],\n", + " ['Hurricane Inez',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Florida Keys'],\n", + " ['Bahamas', 'location.location.contains', \"Walker's Cay\"],\n", + " ['Bahamas', 'location.location.people_born_here', 'Burton P. C. Hall'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc42f9t'],\n", + " ['Southern United States',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Andrew'],\n", + " ['Leevan Sands', 'people.person.place_of_birth', 'Nassau'],\n", + " ['m.0b652s2',\n", + " 'sports.competitor_competition_relationship.competitors',\n", + " 'Derrick Atkins'],\n", + " ['m.0nfblr7',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Tongue of the Ocean', 'common.topic.article', 'm.08wxh0'],\n", + " ['m.0zt_0_q', 'sports.sports_team_roster.player', 'Gavin Christie'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.official_development_assistance',\n", + " 'g.1hhc3cc09'],\n", + " ['m.0khlrs4', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Texas'],\n", + " ['Americas', 'location.location.contains', 'Dominica'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.agriculture_as_percent_of_gdp',\n", + " 'g.1hhc38qjy'],\n", + " ['Nassau', 'common.topic.image', 'Bf-map'],\n", + " ['Pauline Davis-Thompson', 'people.person.gender', 'Female'],\n", + " ['Florida Keys',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Ike'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nf6mv3'],\n", + " ['m.0nf8xdc',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Hurricane Erin', 'meteorology.tropical_cyclone.affected_areas', 'Florida'],\n", + " ['High Rock', 'common.topic.notable_for', 'g.1254z18t8'],\n", + " ['Grand Cay', 'location.location.containedby', 'Bahamas'],\n", + " ['Ivanique Kemp', 'people.person.profession', 'Track and field athlete'],\n", + " ['Hurricane Irene',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Turks and Caicos Islands'],\n", + " ['San Salvador and Rum Cay', 'common.topic.image', 'Map of Bahamas'],\n", + " ['Hurricane Erin', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Atlantic Ocean', 'location.location.contains', 'Anguilla'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4115j'],\n", + " ['David Kelly', 'freebase.valuenotation.is_reviewed', 'Place of death'],\n", + " ['Grand Cay', 'common.topic.article', 'm.0bkcn7'],\n", + " ['Georgia', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Cleo'],\n", + " ['Bahamas', 'meteorology.cyclone_affected_area.cyclones', 'Hurricane Able'],\n", + " ['Blake Bartlett', 'people.person.nationality', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " '1938 New England hurricane'],\n", + " ['Structure', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['United States of America',\n", + " 'location.location.time_zones',\n", + " 'Eastern Time Zone'],\n", + " ['Bahamas', 'common.topic.webpage', 'm.09x2k1q'],\n", + " ['Virginia', 'location.location.time_zones', 'UTC−05:00'],\n", + " ['Bahamas Creole English Language',\n", + " 'language.human_language.main_country',\n", + " 'Bahamas'],\n", + " ['Venris Bennett', 'common.topic.article', 'm.0b__f7h'],\n", + " ['Map of Bahamas', 'common.image.appears_in_topic_gallery', 'Kemps Bay'],\n", + " ['m.0nf8xg_',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1q5jjw5ml'],\n", + " ['Bahamas', 'location.location.people_born_here', 'LaToy Williams'],\n", + " ['Mayaguana', 'common.topic.notable_for', 'g.1255pjsx2'],\n", + " ['Hurricane Bill',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Atlantic Canada'],\n", + " [\"Abraham's Bay\", 'location.location.geolocation', 'm.0czgj_1'],\n", + " ['River Lees', 'common.topic.article', 'm.06w53f6'],\n", + " ['Bahamas national football team',\n", + " 'soccer.football_team.player_statistics',\n", + " 'm.0w9cvtw'],\n", + " ['Caribbean Development Bank', 'common.topic.notable_types', 'Organization'],\n", + " ['Hurricane Noel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Atlantic Canada'],\n", + " ['North Andros', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['m.0j7r7v3', 'sports.sports_team_roster.player', 'Jonathan Barry'],\n", + " ['Lesser Antilles',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Emily'],\n", + " ['Dominica', 'location.country.official_language', 'English Language'],\n", + " ['Alice Town', 'location.location.geolocation', 'm.0clf5jm'],\n", + " ['Bahamas', 'location.location.contains', 'Starve Creek'],\n", + " ['Central Andros', 'location.location.containedby', 'Bahamas'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nf84jd'],\n", + " ['m.0nf84lt',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Bahamas',\n", + " 'location.country.first_level_divisions',\n", + " 'Black Point, Bahamas'],\n", + " [\"Rosie O'Donnell\",\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Hurricane Gordon',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Georgia'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Carolina'],\n", + " ['Airport', 'freebase.type_profile.strict_included_types', 'Location'],\n", + " ['Free National Movement',\n", + " 'organization.organization.geographic_scope',\n", + " 'Bahamas'],\n", + " ['Bahamas', 'location.location.contains', 'Eleuthera'],\n", + " ['Hurricane Arthur',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Atlantic Canada'],\n", + " ['Demitri Knowles', 'people.person.gender', 'Male'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3fshy'],\n", + " ['Hurricane Hazel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'South Carolina'],\n", + " ['Hurricane Hugo',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Guadeloupe'],\n", + " ['m.0kd9w_5', 'olympics.olympic_athlete_affiliation.country', 'Bahamas'],\n", + " ['Cynthia A. Pratt', 'people.person.gender', 'Female'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.prevalence_of_undernourisment',\n", + " 'g.12tb6fzvm'],\n", + " ['Inagua', 'location.administrative_division.country', 'Bahamas'],\n", + " ['South Andros Airport', 'common.topic.notable_types', 'Airport'],\n", + " ['Bahamas',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc45ts1'],\n", + " ['Hurricane Wilma',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Jamaica'],\n", + " ['Laverne Eve', 'people.person.place_of_birth', 'Bahamas'],\n", + " ['Mario Ford', 'cricket.cricket_bowler.pace', 'Medium'],\n", + " ['Newfoundland', 'location.location.containedby', 'Canada'],\n", + " ...],\n", + " [['John Noble', 'film.actor.film', 'm.0h1ldcx'],\n", + " ['m.02_1x1x',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Andy Serkis'],\n", + " ['John Noble', 'people.person.place_of_birth', 'Port Pirie'],\n", + " ['Sean Astin', 'people.person.profession', 'Actor'],\n", + " ['John Noble by Gage Skidmore', 'common.image.size', 'm.0cmnjdl'],\n", + " ['Fringe - Season 4',\n", + " 'tv.tv_series_season.episodes',\n", + " 'One Night in October'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyl5'],\n", + " ['m.04kcrh0', 'tv.regular_tv_appearance.seasons', 'Fringe - Season 1'],\n", + " ['Ian Holm', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Andy Serkis', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['David Wenham',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Transient ischemic attack'],\n", + " ['Location', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['South Australia',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Australia'],\n", + " ['Actor', 'people.profession.specializations', 'Narrator'],\n", + " ['Risen', 'film.film.starring', 'm.0h1ldcx'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gout'],\n", + " ['m.09yhxgh',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Barrett's esophagus\"],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.parent_disease',\n", + " 'Cardiovascular disease'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Prostate cancer'],\n", + " ['Dominic Monaghan', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Fringe', 'award.award_nominated_work.award_nominations', 'm.0b6zl9d'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yq9dfg'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0_lpg5k'],\n", + " ['m.09ywgl5', 'common.webpage.topic', 'John Noble'],\n", + " ['Bernard Hill', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Ian McKellen', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Johnnoble 2003', 'common.image.appears_in_topic_gallery', 'John Noble'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13_1s'],\n", + " ['m.09w22ld', 'common.webpage.topic', 'John Noble'],\n", + " ['Epilepsy', 'medicine.disease.risk_factors', 'Male'],\n", + " ['m.0_glgxt',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Film actor', 'freebase.type_profile.published', 'Published'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3lz1'],\n", + " ['heart attack',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Actor', 'people.profession.specializations', 'Theater Performer'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.07n73w_'],\n", + " ['Voice Actor', 'common.topic.notable_types', 'Profession'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0_lpg7b'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0j7v__g'],\n", + " ['Viggo Mortensen', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " [\"Parkinson's disease\",\n", + " 'medicine.disease.symptoms',\n", + " 'Seborrheic dermatitis'],\n", + " ['Andy Serkis', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.02_98hd',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['m.03l6qx7', 'film.performance.actor', 'John Noble'],\n", + " ['John Madsen', 'tv.tv_character.appeared_in_tv_episodes', 'm.09p3lzf'],\n", + " ['Fringe - Season 4', 'tv.tv_series_season.series', 'Fringe'],\n", + " ['m.0h5jyjp', 'film.performance.character', 'Irving Pichel'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0p70n61'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09ywgl5'],\n", + " ['Liv Tyler', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09ywm7l'],\n", + " ['Andy Serkis', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.02vd0pq', 'film.performance.character', 'Howard Peet'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'John Noble'],\n", + " ['Male', 'common.topic.article', 'm.05zpq8'],\n", + " [\"Scoop: 'Fringe' beaming up Spock!\",\n", + " 'common.resource.annotations',\n", + " 'm.09w22ld'],\n", + " ['Dominic Monaghan', 'people.person.profession', 'Actor'],\n", + " ['m.012zhxfq', 'film.performance.film', 'Superman: Unbound'],\n", + " ['Eddie Thomas', 'film.film_character.portrayed_in_films', 'm.0h1ldcx'],\n", + " ['m.0j13zwq',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyj9'],\n", + " ['m.0h5jyhq', 'film.performance.film', 'SuperFire'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Film actor', 'type.type.expected_by', 'Xxx'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0105ypw8'],\n", + " ['Orlando Bloom', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Parents', 'type.property.reverse_property', 'Children'],\n", + " ['Miranda Otto', 'people.person.profession', 'Actor'],\n", + " ['Fringe - Season 2',\n", + " 'tv.tv_series_season.episodes',\n", + " 'Olivia. In the Lab. With the Revolver.'],\n", + " ['Sleepy Hollow - Season 2',\n", + " 'tv.tv_series_season.regular_cast',\n", + " 'm.0_qp7mv'],\n", + " ['Henry Parrish', 'tv.tv_character.appeared_in_tv_episodes', 'm.011xd9v9'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0zm7bxs'],\n", + " ['Viggo Mortensen', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065j0b'],\n", + " ['m.010g8n89', 'theater.theater_role.play', 'The Substance of Fire'],\n", + " ['The Lord of the Rings: The Two Towers', 'film.film.starring', 'm.0528y98'],\n", + " ['m.0cs3xf3', 'film.performance.actor', 'Samantha Noble'],\n", + " ['Actor', 'common.topic.subject_of', 'Jeremy Yablan'],\n", + " ['m.0b6zl9d', 'award.award_nomination.ceremony', '14th Satellite Awards'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney Stone'],\n", + " ['Film actor', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Bernard Hill', 'people.person.gender', 'Male'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01059z6b'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Sean Bean'],\n", + " ['Actor', 'people.profession.specializations', 'Lifecaster'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d521lm'],\n", + " ['Denethor II', 'film.film_character.portrayed_in_films', 'm.0528y98'],\n", + " ['Actor', 'common.topic.subjects', 'Maria Pitillo'],\n", + " ['Sean Bean', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.0yyv6fs',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Penny Noble', 'freebase.valuenotation.has_value', 'Place of birth'],\n", + " ['Film actor', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Theatre Director',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Roger De Bris'],\n", + " ['m.0pmftcq', 'people.sibling_relationship.sibling', 'Jess Noble'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Liv Tyler'],\n", + " ['Elijah Wood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0h5jyhq', 'film.performance.actor', 'John Noble'],\n", + " ['m.0bnssg0', 'common.webpage.topic', 'John Noble'],\n", + " ['Dominic Monaghan', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01064wv8'],\n", + " ['Voice Actor', 'common.topic.image', 'Atunda Ayenda Cast'],\n", + " [\"Press Tour Diary: 'Fringe'\", 'common.resource.annotations', 'm.09w32cb'],\n", + " ['Ian Holm', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Bernard Hill', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['m.0h5jyk1', 'film.performance.character', 'Mr. Norris'],\n", + " ['Andy Serkis', 'people.person.profession', 'Voice Actor'],\n", + " ['Mr. Norris', 'film.film_character.portrayed_in_films', 'm.0h5jyk1'],\n", + " ['m.010g8n89', 'theater.theater_role.role', 'Isaac Geldhart'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Dominic Monaghan'],\n", + " ['m.0y874gy',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Dupuytren's contracture\"],\n", + " ['m.0h5jykf', 'film.performance.special_performance_type', 'Voice'],\n", + " ['Voice Actor', 'people.profession.specialization_of', 'Actor'],\n", + " ['Hugo Weaving', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['John Noble', 'people.person.profession', 'Actor'],\n", + " ['m.09p3l_l', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Actinic keratosis'],\n", + " ['John Noble', 'people.person.children', 'Jess Noble'],\n", + " ['Transformers Prime Beast Hunters: Predacons Rising',\n", + " 'film.film.starring',\n", + " 'm.0zs55yg'],\n", + " ['m.0ysrsp6', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Hugo Weaving', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Country of nationality', 'type.property.expected_type', 'Country'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3lyp'],\n", + " ['Orlando Bloom', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Dominic Monaghan'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yq9nc6'],\n", + " ['John Noble', 'film.actor.film', 'm.02vd0pq'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065hb5'],\n", + " ['Viggo Mortensen',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Sean Bean', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Fringe - Season 3', 'tv.tv_series_season.regular_cast', 'm.04kcrh0'],\n", + " ['John Noble', 'tv.tv_actor.starring_roles', 'm.04kcrh0'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zg1'],\n", + " ['Australia',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'South Australia'],\n", + " ['Pancreatic cancer',\n", + " 'medicine.icd_9_cm_classification.parent_classification',\n", + " 'Pancreatic cancer'],\n", + " ['Male',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Attention deficit hyperactivity disorder'],\n", + " ['Sean Bean', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011k7hb_'],\n", + " ['m.0yt7bds', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['m.0h5jyx4', 'tv.regular_tv_appearance.character', 'The C.E.O.'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Elijah Wood', 'people.person.profession', 'Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x4_mh'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3l_l'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0ysrs4n'],\n", + " ['Bernard Hill', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Actor', 'people.profession.specializations', 'Camgirl'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gastritis'],\n", + " ['Dominic Monaghan', 'people.person.gender', 'Male'],\n", + " [\"9th Critics' Choice Awards\", 'award.award_ceremony.nominees', 'm.09k3q0p'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Sebaceous cyst'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01012_r3'],\n", + " ['Film actor', 'type.type.properties', 'Film performances'],\n", + " ['Screen Actors Guild',\n", + " 'business.trade_union.professions_represented',\n", + " 'Actor'],\n", + " ['The Mystery of Natalie Wood', 'film.film.starring', 'm.0h5jyjp'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.011xd9wf'],\n", + " [\"Leonard Nimoy and 'Fringe': A match made in geek heaven?\",\n", + " 'common.resource.annotations',\n", + " 'm.09ycb67'],\n", + " ['Port Pirie', 'location.location.containedby', 'Australia'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyfk'],\n", + " ['John Rhys-Davies', 'people.person.profession', 'Voice Actor'],\n", + " ['Henry Parrish', 'tv.tv_character.appeared_in_tv_program', 'm.0_qp7mv'],\n", + " ['m.0vx90yr', 'film.performance.actor', 'Samantha Noble'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Viggo Mortensen'],\n", + " ['Ian Holm', 'people.person.gender', 'Male'],\n", + " ['Person', 'type.type.properties', 'Gender'],\n", + " ['Fringe - Season 2', 'tv.tv_series_season.episodes', 'Of Human Action'],\n", + " ['m.04kcrh0', 'tv.regular_tv_appearance.seasons', 'Fringe - Season 2'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jykt'],\n", + " ['Theatre Director',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Mangiafuoco'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Viggo Mortensen'],\n", + " ['Elijah Wood', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02_98hd'],\n", + " ['Walter Bishop', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Orlando Bloom'],\n", + " ['Female', 'common.topic.subjects', 'ZasPorn'],\n", + " ['Child Actor', 'people.profession.specialization_of', 'Actor'],\n", + " ['John Noble', 'people.person.gender', 'Male'],\n", + " ['South Australia', 'location.australian_state.capital_city', 'Adelaide'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0y8734m'],\n", + " ['Viggo Mortensen', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Dominic Monaghan', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Voice acting in Japan',\n", + " 'people.profession.specialization_of',\n", + " 'Voice Actor'],\n", + " ['Actor', 'common.topic.image', 'Flickr 6130489'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'John Rhys-Davies'],\n", + " ['m.0h5jyjp', 'film.performance.film', 'The Mystery of Natalie Wood'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0zdc1gp'],\n", + " ['Hugo Weaving',\n", + " 'medicine.notable_person_with_medical_condition.condition',\n", + " 'Epilepsy'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d5218t'],\n", + " ['m.09ycb67',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.0_lpg7b',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Actor',\n", + " 'base.skosbase.vocabulary_equivalent_topic.equivalent_concept',\n", + " 'Actors'],\n", + " ['Port Pirie',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Solomontown Beach'],\n", + " ['Irving Pichel', 'film.film_character.portrayed_in_films', 'm.0h5jyjp'],\n", + " ['John Noble',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0gxvv0q',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Emily Condon'],\n", + " ['Film actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010h52h5'],\n", + " ['Entrepreneur', 'common.topic.subject_of', 'Michael Palance'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Corneal abrasion'],\n", + " ['m.09ywm7l',\n", + " 'common.webpage.resource',\n", + " \"'Fringe' finale: Did it go too far? Not far enough? Just right? Some further thoughts...\"],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64q8'],\n", + " ['m.02zd4tr',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Miranda Otto', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Ian McKellen', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Ian McKellen',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['John Noble', 'film.actor.film', 'm.03l6qx7'],\n", + " ['m.09ysvx8', 'common.webpage.topic', 'Voice Actor'],\n", + " ['Cate Blanchett', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0116r92_'],\n", + " ['Sean Astin', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney cancer'],\n", + " ['Liv Tyler', 'people.person.gender', 'Female'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Hugo Weaving', 'people.person.nationality', 'Australia'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Sean Bean'],\n", + " ['Penny Noble', 'people.person.gender', 'Female'],\n", + " ['Liv Tyler', 'common.topic.notable_types', 'Film actor'],\n", + " [\"'Lost' writer to run new J.J. Abrams drama 'Fringe'\",\n", + " 'common.resource.annotations',\n", + " 'm.09w6tcg'],\n", + " [\"'Fringe' exclusive: It's Walter vs. William in the finale!\",\n", + " 'common.resource.annotations',\n", + " 'm.09w4nn7'],\n", + " ['m.0h5jyj9', 'film.performance.character', 'Dr. Richards'],\n", + " ['Fringe - Season 2', 'tv.tv_series_season.episodes', 'The Bishop Revival'],\n", + " ['m.04kcrh0', 'tv.regular_tv_appearance.actor', 'John Noble'],\n", + " ['Billy Boyd', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['m.09y5k14', 'common.webpage.topic', 'John Noble'],\n", + " ['The Dragon Spirit', 'film.film_character.portrayed_in_films', 'm.0h5jykf'],\n", + " ['Jess Noble', 'freebase.valuenotation.has_value', 'Place of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01059w7q'],\n", + " ['Cate Blanchett', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0zdj7py'],\n", + " ['Andy Serkis', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['heart attack', 'medicine.disease.risk_factors', 'Male'],\n", + " ['David Wenham', 'people.person.profession', 'Actor'],\n", + " ['m.0znpr5t',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['John Rhys-Davies',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.011xd9wf', 'tv.tv_guest_role.character', 'Henry Parrish'],\n", + " ['From', 'type.property.expected_type', 'Date/Time'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Urolithiasis'],\n", + " ['Phone Trick',\n", + " 'base.thoroughbredracing.thoroughbred_racehorse.sex',\n", + " 'Male'],\n", + " ['Miranda Otto', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x5157'],\n", + " ['Daniel Noble', 'people.person.parents', 'John Noble'],\n", + " ['Him/Herself',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zhf'],\n", + " ['Ian McKellen', 'people.person.profession', 'Actor'],\n", + " ['Film actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.011skjd7'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.010g_0js'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Ian Holm'],\n", + " ['Person', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Samantha Noble', 'film.actor.film', 'm.0cs3xf3'],\n", + " ['Cate Blanchett',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0zs55yg',\n", + " 'film.performance.film',\n", + " 'Transformers Prime Beast Hunters: Predacons Rising'],\n", + " ['m.0h1ldcx', 'film.performance.film', 'Risen'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.includes_causes_of_death',\n", + " 'Myocardial Ischemia'],\n", + " ['Actor', 'people.profession.specializations', 'Pornographic film actor'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Hair loss'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Liv Tyler', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.0gxvv0q'],\n", + " ['Adelaide', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['The Mule', 'film.film.country', 'Australia'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0z3vx9q'],\n", + " ['Satellite Award for Best Supporting Actor – Series, Miniseries or Television Film',\n", + " 'award.award_category.nominees',\n", + " 'm.0b6zl9d'],\n", + " ['Adelaide', 'location.location.containedby', 'South Australia'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011nhmrw'],\n", + " ['Dominic Monaghan', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.0kfyrv7',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['Sean Astin', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['John Noble', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['m.09ykckh', 'common.webpage.topic', 'John Noble'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Syphilis'],\n", + " ['Ian Holm', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['m.04kcrh0', 'tv.regular_tv_appearance.seasons', 'Fringe - Season 4'],\n", + " [\"Parkinson's disease\", 'medicine.disease.risk_factors', 'Male'],\n", + " ['Fracture', 'film.film.starring', 'm.02vd0pq'],\n", + " ['Theatre Director', 'common.topic.notable_for', 'g.1257mp68p'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Hirschsprung's disease\"],\n", + " ['Stroke', 'medicine.disease.risk_factors', 'Transient ischemic attack'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Multiple myeloma'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Ian McKellen'],\n", + " ['Ian McKellen', 'people.person.profession', 'Voice Actor'],\n", + " ['Voice Actor',\n", + " 'people.profession.specializations',\n", + " 'Voice acting in Japan'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Myocardial Ischemia'],\n", + " ['Ian McKellen', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Liv Tyler', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Bernard Hill', 'common.topic.notable_types', 'Film actor'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['m.0yj_j02', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Sean Bean', 'people.person.profession', 'Actor'],\n", + " ['m.0yvmvhx', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Narrator', 'common.topic.notable_types', 'Profession'],\n", + " ['Liv Tyler', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['m.0ggdnq9',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Olivia. In the Lab. With the Revolver.'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Miranda Otto'],\n", + " ['Sergeant', 'film.film_character.portrayed_in_films', 'm.0h5jyl5'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0zm9_69'],\n", + " ['Sean Bean', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x4_6k'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pancreatic cancer'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Anatoly Markov', 'tv.tv_character.appeared_in_tv_program', 'm.0h5jyw6'],\n", + " ['Andy Serkis', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Ian Holm', 'people.person.profession', 'Voice Actor'],\n", + " ['John Noble', 'people.person.children', 'Samantha Noble'],\n", + " ['m.03jpb4c', 'film.performance.film', 'One Night with the King'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvjy3'],\n", + " ['Dr. Salinger', 'tv.tv_character.appeared_in_tv_episodes', 'm.09p3l_6'],\n", + " ['m.0h5jyfk', 'film.performance.actor', 'John Noble'],\n", + " ['Film actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0118g900'],\n", + " ['m.0h5jyj9', 'film.performance.actor', 'John Noble'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Location'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'David Wenham'],\n", + " ['John Rhys-Davies', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0bm6pqb'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09w22ld'],\n", + " ['Karl Urban', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09ykckh'],\n", + " ['Maria Pitillo', 'people.person.gender', 'Female'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64rh'],\n", + " ['Children', 'rdf-schema#domain', 'Person'],\n", + " ['m.0pmftv_', 'people.sibling_relationship.sibling', 'Daniel Noble'],\n", + " [\"Critics' Choice Movie Award for Best Acting Ensemble\",\n", + " 'award.award_category.nominees',\n", + " 'm.09k3q0p'],\n", + " ['m.0h5jywd', 'tv.regular_tv_appearance.character', 'Dunston'],\n", + " ['Bernard Hill', 'people.person.profession', 'Voice Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01012_bc'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x4_4k'],\n", + " ['m.0h5jyfk', 'film.performance.film', 'Virtual Nightmare'],\n", + " ['Actor', 'type.property.expected_type', 'Film actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x5045'],\n", + " ['Esophageal cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Daniel Noble', 'common.topic.notable_types', 'Person'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065h_j'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'John Rhys-Davies'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jygl'],\n", + " ['Sean Astin', 'common.topic.notable_types', 'Film actor'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Esophageal cancer'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0101341c'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Peter Moyes'],\n", + " ['Port Pirie',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Recorder'],\n", + " ['Film performances', 'rdf-schema#domain', 'Film actor'],\n", + " ['Film actor', 'type.type.expected_by', 'Film actor'],\n", + " ['Hugo Weaving', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Billy Boyd', 'people.person.profession', 'Actor'],\n", + " ['m.09y3wpk',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Him/Herself',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zzn'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Ian McKellen'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yss2hv'],\n", + " ['Film actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.011cm1z3'],\n", + " ['m.09p3l_l',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Pandemonium, Part 2'],\n", + " ['m.0h5jxy8', 'people.marriage.spouse', 'Penny Noble'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zzn'],\n", + " ['m.0znpr6h',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Hugh Cairns'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0115s8v4'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Liv Tyler'],\n", + " ['Daniel Noble', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['Port Pirie', 'location.location.containedby', 'South Australia'],\n", + " ['John Noble', 'film.actor.film', 'm.010h3gm8'],\n", + " ['m.0h5jxy8', 'freebase.valuenotation.has_no_value', 'To'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011k7h9v'],\n", + " ['Billy Boyd', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Ian McKellen', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Amyotrophic lateral sclerosis'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " 'Screen Actors Guild'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0zdb4km'],\n", + " ['Port Pirie', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'John Noble'],\n", + " ['Male',\n", + " 'base.skosbase.vocabulary_equivalent_topic.equivalent_concept',\n", + " 'Males'],\n", + " ['m.05bvk3z',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['Actor', 'people.profession.specializations', 'Softcore Actor'],\n", + " ['Him/Herself',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zwq'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13_18'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zzb'],\n", + " ['Fringe - Season 5', 'tv.tv_series_season.series', 'Fringe'],\n", + " ['m.0528y98', 'film.performance.actor', 'John Noble'],\n", + " ['Location', 'type.type.properties', 'People born here'],\n", + " ['Maria Pitillo', 'people.person.profession', 'Entrepreneur'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01064v8s'],\n", + " ['m.064tkhd', 'base.events.performance.type_of_performance', 'Actor'],\n", + " ['m.0h5jygl', 'film.performance.actor', 'John Noble'],\n", + " ['Daniel Noble', 'people.person.profession', 'Graphic Artist'],\n", + " ['The Mystery of Natalie Wood', 'film.film.country', 'Australia'],\n", + " ['Actor', 'people.profession.specializations', 'Audio Dramatist'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Miranda Otto', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['SuperFire', 'film.film.starring', 'm.0h5jyhq'],\n", + " ['m.09p3l_6', 'tv.tv_guest_role.character', 'Dr. Salinger'],\n", + " ['m.0b4d5rz',\n", + " 'award.award_nomination.nominated_for',\n", + " 'The Lord of the Rings: The Return of the King'],\n", + " ['m.0cf_87h', 'film.performance.actor', 'John Noble'],\n", + " ['m.0b6zlcc', 'award.award_nomination.ceremony', '13th Satellite Awards'],\n", + " ['Bernard Hill', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0cq5xwp', 'common.webpage.category', 'Curated Topic'],\n", + " ['Female', 'medicine.risk_factor.diseases', 'Kyphosis'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011skhxv'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yv3vs2'],\n", + " ['m.0zdb4km',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yt7n52'],\n", + " ['Actor', 'people.profession.specializations', 'Monologist'],\n", + " ['Ian McKellen', 'common.topic.notable_types', 'Film actor'],\n", + " ['SFX Award for Best Actor',\n", + " 'award.award_category.disciplines_or_subjects',\n", + " 'Male'],\n", + " ['m.0j13zzn',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Ian Holm'],\n", + " ['Karl Urban', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Narrator',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zvz'],\n", + " ['m.0h5jygz', 'film.performance.character', 'Fergus Hunter'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04mmfr8'],\n", + " ['Sean Bean', 'people.person.profession', 'Voice Actor'],\n", + " ['m.0pmftcq', 'people.sibling_relationship.sibling', 'Samantha Noble'],\n", + " ['John Rhys-Davies', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0zs55yg', 'film.performance.actor', 'John Noble'],\n", + " ['m.09y3s8z',\n", + " 'common.webpage.resource',\n", + " \"J.J. Abrams gives Comic-Con attendees first peek at 'Fringe'\"],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.0kfyrt3'],\n", + " ['Film actor', 'freebase.type_profile.kind', 'Title'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.0hhzbbf'],\n", + " ['Andy Serkis', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Cardiovascular disease'],\n", + " ['m.0zm7bxs', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Cate Blanchett', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Sean Astin'],\n", + " ['Actor', 'common.topic.subjects', 'Chris Free'],\n", + " ['Fringe - Season 2', 'tv.tv_series_season.regular_cast', 'm.04kcrh0'],\n", + " ['m.011xd9v_', 'tv.tv_guest_role.character', 'Henry Parrish'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pyloric stenosis'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Hugo Weaving'],\n", + " ['Fringe - Season 5', 'tv.tv_series_season.regular_cast', 'm.04kcrh0'],\n", + " ['Billy Boyd', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Miranda Otto',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Miranda Otto'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvkm6'],\n", + " ['John Noble', 'people.person.profession', 'Voice Actor'],\n", + " ['Jess Noble', 'people.person.parents', 'Penny Noble'],\n", + " ['Children', 'type.property.expected_type', 'Person'],\n", + " ['Actor', 'common.topic.subject_of', 'Nakshatra Bagwe'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.0g0h_jy'],\n", + " ['John Noble', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.0blp5sr', 'biology.hybrid_parentage.parent_sex', 'Male'],\n", + " ['Place of birth', 'type.property.schema', 'Person'],\n", + " ['m.03l6qx7', 'film.performance.character', 'Denethor II'],\n", + " ['Miranda Otto', 'people.person.nationality', 'Australia'],\n", + " ['Theatre Director', 'common.topic.webpage', 'm.09y22z_'],\n", + " ['Hugo Weaving', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Daniel Noble', 'people.person.sibling_s', 'm.0pmfsm_'],\n", + " ['John Noble', 'tv.tv_actor.starring_roles', 'm.0h5jyws'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Mark Jamar'],\n", + " ['Sean Bean', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Theatre Director',\n", + " 'people.profession.specializations',\n", + " 'Artistic Director'],\n", + " ['Hugo Weaving', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Sean Astin', 'people.person.gender', 'Male'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0106506g'],\n", + " ['Film actor', 'type.type.expected_by', 'Actor Name'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0105ytrx'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.011xd9v9'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Benign prostatic hyperplasia'],\n", + " ['Ian McKellen', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Karl Urban',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Karl Urban', 'people.person.gender', 'Male'],\n", + " ['Actor', 'base.schemastaging.context_name.pronunciation', 'g.125_ldmrv'],\n", + " ['Actor', 'rdf-schema#range', 'Film actor'],\n", + " ['One Night with the King', 'film.film.starring', 'm.03jpb4c'],\n", + " ['Liv Tyler', 'people.person.profession', 'Actor'],\n", + " ['John Noble', 'award.award_nominee.award_nominations', 'm.0b6zlcc'],\n", + " ['South Australia', 'location.location.containedby', 'Australia'],\n", + " ['John Noble', 'film.actor.film', 'm.0528y98'],\n", + " ['Paul Baylis', 'film.film_character.portrayed_in_films', 'm.0h5jyhq'],\n", + " ['m.09p3lzf',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Bending and Breaking'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0105cpdr'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3lzf'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yqsv26'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " \"Actors' Equity Association\"],\n", + " ['m.09ywgl5',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Stroke'],\n", + " ['m.0h5jxy8', 'freebase.valuenotation.has_value', 'From'],\n", + " ['Hugo Weaving', 'people.person.profession', 'Voice Actor'],\n", + " ['Actor', 'common.topic.notable_types', 'Profession'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0105dqvw'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Geoffrey Reed'],\n", + " ['Miranda Otto', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0z3wm28'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Leprosy'],\n", + " ['Samantha Noble', 'common.topic.article', 'm.043rnx7'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Sam Mayes'],\n", + " ['Superman: Unbound', 'film.film.starring', 'm.012zhxfq'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0ydy57_'],\n", + " ['m.0n9wzxp',\n", + " 'tv.regular_tv_appearance.special_performance_type',\n", + " 'Him/Herself'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09w32cb'],\n", + " [\"J.J. Abrams gives Comic-Con attendees first peek at 'Fringe'\",\n", + " 'common.resource.annotations',\n", + " 'm.09y3s8z'],\n", + " [\"Actors' Equity Association\",\n", + " 'business.trade_union.professions_represented',\n", + " 'Actor'],\n", + " ['Maria Pitillo', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Amyotrophic lateral sclerosis', 'medicine.disease.risk_factors', 'Male'],\n", + " ['m.09p3lz1', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Cate Blanchett', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Anatoly Markov', 'tv.tv_character.appeared_in_tv_episodes', 'm.09p3lyp'],\n", + " ['Dominic Monaghan', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Male', 'common.topic.image', 'male.jpg'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011jy0zd'],\n", + " ['Billy Boyd', 'people.person.gender', 'Male'],\n", + " ['m.0n7xsws',\n", + " 'award.award_honor.honored_for',\n", + " 'The Lord of the Rings: The Return of the King'],\n", + " ['m.0h5jygz', 'film.performance.film', 'The Outsider'],\n", + " ['Port Pirie', 'common.topic.notable_for', 'g.125dhx05z'],\n", + " ['Actor', 'common.topic.subjects', 'Bleona'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Abdominal aortic aneurysm'],\n", + " ['John Noble', 'film.actor.film', 'm.0cf_87h'],\n", + " ['Liv Tyler', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01076n1z'],\n", + " ['m.0ncn2bq', 'tv.tv_regular_personal_appearance.appearance_type', 'Host'],\n", + " ['Epilepsy',\n", + " 'medicine.disease.notable_people_with_this_condition',\n", + " 'Hugo Weaving'],\n", + " ['Cardiovascular disease',\n", + " 'medicine.disease_cause.diseases',\n", + " 'Transient ischemic attack'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.0gd11j5'],\n", + " ['Spokesperson', 'common.topic.notable_types', 'Profession'],\n", + " ['heart attack',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Abdominal aortic aneurysm'],\n", + " ['Elijah Wood', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.07n73w_',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Billy Boyd'],\n", + " ['Sean Astin', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Billy Boyd'],\n", + " ['m.0j13_2t',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011jy0y9'],\n", + " ['m.09k3q0p',\n", + " 'award.award_nomination.ceremony',\n", + " \"9th Critics' Choice Awards\"],\n", + " ['m.0j13zhf',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Epilepsy'],\n", + " ['Actor',\n", + " 'award.award_discipline.awards_in_this_discipline',\n", + " 'Teddy Award for Best Acting Performance'],\n", + " ['Viggo Mortensen', 'people.person.profession', 'Actor'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Alcohol abuse'],\n", + " ['Actor', 'owl#inverseOf', 'Film performances'],\n", + " ['The C.E.O.', 'tv.tv_character.appeared_in_tv_program', 'm.0h5jyx4'],\n", + " ['Massimo Zanini', 'freebase.domain_profile.category', 'Male'],\n", + " ['Australia',\n", + " 'location.country.administrative_divisions',\n", + " 'South Australia'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Elijah Wood'],\n", + " ['Karl Urban', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['m.0h5jyk1', 'film.performance.film', \"The Monkey's Mask\"],\n", + " ['m.0bnssg0', 'common.webpage.in_index', 'Blissful Celebrities'],\n", + " ['Sleepy Hollow', 'tv.tv_program.seasons', 'Sleepy Hollow - Season 2'],\n", + " ['Hugo Weaving', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yj_j02'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.010wvs8m'],\n", + " ['Phone Trick', 'biology.organism.sex', 'Male'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zhf'],\n", + " ['John Noble', 'common.topic.article', 'm.02fgmg'],\n", + " ['Miranda Otto', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Agent'],\n", + " ['Samantha Noble', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0pmftv_', 'people.sibling_relationship.sibling', 'Jess Noble'],\n", + " ['Bernard Hill', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.09p3l_6', 'tv.tv_guest_role.episodes_appeared_in', 'Epiphany'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09y5k14'],\n", + " ['10th Screen Actors Guild Awards',\n", + " 'award.award_ceremony.nominees',\n", + " 'm.0b4d5rz'],\n", + " ['Fringe', 'tv.tv_program.seasons', 'Fringe - Season 5'],\n", + " ['m.0n9wzxp', 'tv.regular_tv_appearance.actor', 'John Noble'],\n", + " ['m.0j13_18',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['m.0b6zlcc',\n", + " 'award.award_nomination.award',\n", + " 'Satellite Award for Best Supporting Actor – Series, Miniseries or Television Film'],\n", + " ['Sean Bean', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Date of birth', 'rdf-schema#range', 'Date/Time'],\n", + " ['Cate Blanchett', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Actor', 'freebase.equivalent_topic.equivalent_type', 'Film actor'],\n", + " ['m.0h5jygl', 'film.performance.character', 'General Booth'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Inguinal hernia'],\n", + " ['m.012zhxfq', 'film.performance.actor', 'John Noble'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Andy Serkis'],\n", + " ['Cate Blanchett', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['Daniel Noble',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Country of nationality'],\n", + " ['Sean Astin', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['Parents', 'rdf-schema#domain', 'Person'],\n", + " ['m.0ncn2bq', 'tv.tv_regular_personal_appearance.person', 'John Noble'],\n", + " ['Fringe - Season 4', 'tv.tv_series_season.regular_cast', 'm.04kcrh0'],\n", + " ['Samantha Noble', 'common.topic.notable_for', 'g.12558l1ff'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Karl Urban'],\n", + " ['To', 'type.property.expected_type', 'Date/Time'],\n", + " ['Jeremy Yablan', 'people.person.gender', 'Male'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0ysrsp6'],\n", + " ['m.010wvs8m',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['To', 'rdf-schema#range', 'Date/Time'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Dominic Monaghan'],\n", + " ['Ivan Yugorsky', 'film.film_character.portrayed_in_films', 'm.0cf_87h'],\n", + " ['m.0h9g3_2', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Location of ceremony', 'type.property.expected_type', 'Location'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Elijah Wood'],\n", + " ['Voice Actor', 'common.topic.webpage', 'm.09ysvx8'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0107yfyq'],\n", + " ['m.0pmfsm_', 'people.sibling_relationship.sibling', 'Daniel Noble'],\n", + " ['Running Scared', 'film.film.starring', 'm.0cf_87h'],\n", + " ['Theatre Director',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Orson Welles'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " \"Chennai Actors' Association\"],\n", + " ['Parents', 'type.property.expected_type', 'Person'],\n", + " ['Jeremy Yablan', 'people.person.profession', 'Actor'],\n", + " ['m.0j13zvz',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Narrator'],\n", + " ['m.011xd9wp', 'tv.tv_guest_role.character', 'Henry Parrish'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'heart attack'],\n", + " ['Hugo Weaving', 'people.person.gender', 'Male'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0114x8fq'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Cate Blanchett'],\n", + " ['10th Screen Actors Guild Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0n7xsws'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zvz'],\n", + " ['John Noble', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['The Lord of the Rings: The Return of the King',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.09k3pgy'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01012_4j'],\n", + " ['Daniel Noble', 'people.person.profession', 'Entrepreneur'],\n", + " ['Penny Noble', 'people.person.spouse_s', 'm.0h5jxy8'],\n", + " ['m.03jpb4c', 'film.performance.actor', 'John Noble'],\n", + " ['Port Pirie', 'common.topic.article', 'm.02fgsz'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011lztln'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yssql4'],\n", + " ['Call Me Mr. Brown', 'film.film.country', 'Australia'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.0kfyrvl'],\n", + " ['Gender', 'rdf-schema#range', 'Gender'],\n", + " ['m.09w6tcg',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['John Rhys-Davies', 'common.topic.notable_types', 'Film actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.010km078'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Ian Holm'],\n", + " ['m.09y5k14',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Jess Noble', 'freebase.valuenotation.has_value', 'Country of nationality'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j14xl5'],\n", + " ['m.09yhxgh', 'common.webpage.topic', 'Voice Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01076mh8'],\n", + " ['John Noble', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Golden Globe Award for Best Actress – Motion Picture – Musical or Comedy',\n", + " 'award.award_category.disciplines_or_subjects',\n", + " 'Actor'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Hypospadias'],\n", + " ['Actor', 'common.topic.notable_for', 'g.125fmxc48'],\n", + " ['Samantha Noble',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Film actor', 'type.type.expected_by', 'age'],\n", + " ['Place of birth', 'rdf-schema#range', 'Location'],\n", + " ['Dr. Richards', 'film.film_character.portrayed_in_films', 'm.0h5jyj9'],\n", + " ['Jess Noble', 'people.person.parents', 'John Noble'],\n", + " ['Pat Shepherd', 'film.film_character.portrayed_in_films', 'm.010h3gm8'],\n", + " ['Viggo Mortensen', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['Person', 'type.type.properties', 'Country of nationality'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Location'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x4zfq'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Bladder cancer'],\n", + " ['Person',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Physically Instantiable'],\n", + " ['Jess Noble', 'people.person.profession', 'Journalist'],\n", + " ['m.0h5jyfk', 'film.performance.character', 'Dad'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0ys_bxl'],\n", + " ['John Noble', 'tv.tv_actor.starring_roles', 'm.0_qp7mv'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.0105drkb'],\n", + " ['City/Town/Village', 'freebase.type_hints.included_types', 'Location'],\n", + " ['m.05bvjyj',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Female'],\n", + " ['Maria Pitillo', 'common.topic.notable_types', 'Film actor'],\n", + " ['Billy Boyd', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.0h5jyws', 'tv.regular_tv_appearance.character', 'Dr. Helpman'],\n", + " ['Theatre Director',\n", + " 'base.events.type_of_performance.performances_of_this_type',\n", + " 'm.064tkhd'],\n", + " ['m.0b4d5rz',\n", + " 'award.award_nomination.ceremony',\n", + " '10th Screen Actors Guild Awards'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3l_6'],\n", + " ['Virtual Nightmare', 'film.film.starring', 'm.0h5jyfk'],\n", + " ['Film actor', 'freebase.type_hints.included_types', 'Person'],\n", + " ['m.0zdc1gp',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['m.0bnssg0', 'common.webpage.in_index', 'Blissful Master Index'],\n", + " ['Him/Herself',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13_1s'],\n", + " ['Male', 'sports.sports_gender.sports_teams', 'Vive Targi Kielce'],\n", + " ['Prostate cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['m.0n7xsws',\n", + " 'award.award_honor.ceremony',\n", + " '10th Screen Actors Guild Awards'],\n", + " ['Ian Holm', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Miranda Otto', 'common.topic.notable_types', 'Film actor'],\n", + " ['m.09td39b',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.010ggv9n'],\n", + " ['Film actor', 'type.type.domain', 'Film'],\n", + " ['The Last Airbender', 'film.film.starring', 'm.0h5jykf'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yqwsc9'],\n", + " ['Person', 'type.type.properties', 'Children'],\n", + " ['m.0j13zg1',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Him/Herself'],\n", + " ['Theatre Director', 'common.topic.image', 'Martyrdom of st Appolonia'],\n", + " ['Andy Serkis', 'people.person.gender', 'Male'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0svq1x2'],\n", + " ['Sean Bean',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Hugo Weaving', 'common.topic.notable_types', 'Film actor'],\n", + " ['The Lord of the Rings: The Return of the King',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.09k3q0p'],\n", + " ['m.0yq9nc6', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Film actor', 'type.type.expected_by', 'Actor'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Ian McKellen'],\n", + " ['Dr. Helpman', 'tv.tv_character.appeared_in_tv_program', 'm.0h5jyws'],\n", + " ['Journalist', 'common.topic.notable_types', 'Profession'],\n", + " ['Actor', 'people.profession.specializations', 'Child Actor'],\n", + " ['Orlando Bloom',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Mangiafuoco', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Film actor', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0101339f'],\n", + " ['Stroke', 'medicine.risk_factor.diseases', 'Epilepsy'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0hgtcbw'],\n", + " ['m.0w5j7dr', 'education.education.student', 'Samantha Noble'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " \"South Indian Film Artistes' Association\"],\n", + " ['Mangiafuoco',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Theatre Director'],\n", + " ['Voice Actor', 'common.topic.webpage', 'm.09yhxgh'],\n", + " ['The Nostradamus Kid', 'film.film.starring', 'm.0h5jygl'],\n", + " ['m.0d521lm', 'biology.hybrid_parentage.parent_sex', 'Male'],\n", + " ['Actor', 'common.topic.subject_of', 'Chris Free'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " 'Kerala Actors Association'],\n", + " ['m.09ywm7l', 'common.webpage.topic', 'John Noble'],\n", + " ['John Noble', 'people.person.profession', 'Theatre Director'],\n", + " ['Jeremy Yablan', 'common.topic.subject_of', 'Actor'],\n", + " ['m.09w6tcg', 'common.webpage.topic', 'John Noble'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Sean Astin'],\n", + " ['The Dreaming', 'film.film.country', 'Australia'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'David Wenham'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0blp5sr'],\n", + " ['m.09ysvx8',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Actor',\n", + " 'people.profession.represented_by_trade_unions',\n", + " \"Canadian Actors' Equity Association\"],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Autism'],\n", + " ['Samantha Noble', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyk1'],\n", + " ['m.0ys_67l', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['m.0gy5czt', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Henry Parrish', 'tv.tv_character.appeared_in_tv_episodes', 'm.011xd9wp'],\n", + " ['John Noble', 'common.topic.notable_for', 'g.1255xl72k'],\n", + " ['The Dreaming', 'film.film.starring', 'm.0h5jyj9'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.includes_causes_of_death',\n", + " 'heart attack'],\n", + " ['heart attack', 'medicine.disease.symptoms', 'Ventricular tachycardia'],\n", + " ['m.09w4nn7',\n", + " 'common.webpage.resource',\n", + " \"'Fringe' exclusive: It's Walter vs. William in the finale!\"],\n", + " ['m.0hhzbbf',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'One Night in October'],\n", + " ['Actor', 'common.topic.subjects', 'Aaron Morris'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.0h9g3_2'],\n", + " ['Dunston', 'tv.tv_character.appeared_in_tv_program', 'm.0h5jywd'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Cate Blanchett'],\n", + " ['Place of birth', 'rdf-schema#domain', 'Person'],\n", + " ['m.09k3q0p',\n", + " 'award.award_nomination.nominated_for',\n", + " 'The Lord of the Rings: The Return of the King'],\n", + " ['Theatre Director',\n", + " 'freebase.equivalent_topic.equivalent_type',\n", + " 'Theater Director'],\n", + " ['Pancreatic cancer',\n", + " 'medicine.icd_9_cm_classification.includes_classifications',\n", + " 'Pancreatic cancer'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.011xd9wp'],\n", + " ['Samantha Noble', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Stroke', 'medicine.disease.risk_factors', 'Hair loss'],\n", + " ['Samantha Noble', 'film.actor.film', 'm.0cg67s4'],\n", + " ['Actor', 'people.profession.specializations', 'Theater Actress'],\n", + " ['m.09y22z_', 'common.webpage.topic', 'Theatre Director'],\n", + " ['Date of birth', 'type.property.schema', 'Person'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Paget's disease of bone\"],\n", + " ['Cate Blanchett', 'people.person.gender', 'Female'],\n", + " ['m.09y22z_',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Elijah Wood', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Meurik', 'tv.tv_character.appeared_in_tv_episodes', 'm.09p3lz1'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.010ggvby'],\n", + " ['Fringe', 'tv.tv_program.seasons', 'Fringe - Season 4'],\n", + " ['Viggo Mortensen', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"9th Critics' Choice Awards\",\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.09k3pgy'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Billy Boyd'],\n", + " ['Daniel Noble', 'people.person.parents', 'Penny Noble'],\n", + " ['Theatre Director', 'common.topic.article', 'm.0q04n'],\n", + " ['Voice Actor', 'film.film_job.films_with_this_crew_job', 'm.0yqsv5c'],\n", + " ['m.0w2y671',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['m.09ywm7l',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0yyv6fs'],\n", + " ['Penny Noble', 'common.topic.notable_types', 'Person'],\n", + " ['Orlando Bloom', 'people.person.gender', 'Male'],\n", + " ['m.0yq9fxd', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['m.09w6tcg',\n", + " 'common.webpage.resource',\n", + " \"'Lost' writer to run new J.J. Abrams drama 'Fringe'\"],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Sean Bean'],\n", + " ['Andy Serkis', 'common.topic.notable_types', 'Film actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01188fdq'],\n", + " ['Attention deficit hyperactivity disorder',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Alcohol abuse'],\n", + " ['m.0_qp7mv', 'tv.regular_tv_appearance.series', 'Sleepy Hollow'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0109n_mn'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01076m3w'],\n", + " ['John Rhys-Davies', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.0h5jyhq', 'film.performance.character', 'Paul Baylis'],\n", + " ['Place of birth', 'type.property.reverse_property', 'People born here'],\n", + " ['Jeremy Yablan', 'common.topic.subjects', 'Actor'],\n", + " ['Voice Actor',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Jenny Larson'],\n", + " ['Ian McKellen', 'people.person.gender', 'Male'],\n", + " ['John Noble', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0gxvt_4',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_vkgyr'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Liv Tyler'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Orlando Bloom'],\n", + " ['Person', 'freebase.type_profile.published', 'Published'],\n", + " ['The Lord of the Rings: The Return of the King',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0n7xsws'],\n", + " ['Actor',\n", + " 'award.award_discipline.awards_in_this_discipline',\n", + " 'Palm Dog Award'],\n", + " ['Sean Astin', 'people.person.profession', 'Voice Actor'],\n", + " ['Actor', 'common.topic.subject_of', 'Inessa Kraft'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Cate Blanchett'],\n", + " ['Samantha Noble', 'people.person.parents', 'Penny Noble'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.0ggdnq9'],\n", + " ['m.0b6zlcc', 'award.award_nomination.award_nominee', 'John Noble'],\n", + " ['Karl Urban', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['Elijah Wood', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x4_7p'],\n", + " ['John Rhys-Davies', 'people.person.profession', 'Actor'],\n", + " ['Actor', 'people.profession.specializations', 'Silent Film Actress'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Retroperitoneal fibrosis'],\n", + " ['Liv Tyler', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.09y0jjp', 'common.webpage.topic', 'Voice Actor'],\n", + " ['Samantha Noble', 'people.person.profession', 'Model'],\n", + " ['Sean Astin', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Denethor II', 'film.film_character.portrayed_in_films', 'm.03l6qx7'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Anaplastic thyroid cancer'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065h1j'],\n", + " ['The Lord of the Rings: The Return of the King',\n", + " 'film.film.starring',\n", + " 'm.03l6qx7'],\n", + " ['m.012sb5xr', 'business.employment_tenure.title', 'Actor'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0blp580'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Andy Serkis'],\n", + " ['Ian Holm', 'people.person.profession', 'Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01059z4k'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0107m19w'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Sean Bean'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Viggo Mortensen'],\n", + " ['m.012nj6qx',\n", + " 'education.education.major_field_of_study',\n", + " 'Theatre Director'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'David Tiller'],\n", + " [\"Parkinson's disease\",\n", + " 'medicine.icd_9_cm_classification.parent_classification',\n", + " \"Parkinson's disease\"],\n", + " ['m.0j13_2t',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Him/Herself'],\n", + " ['David Wenham', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.0h5jyj9', 'film.performance.film', 'The Dreaming'],\n", + " ['Him/Herself',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0j13zg1'],\n", + " [\"'Fringe' finale: Did it go too far? Not far enough? Just right? Some further thoughts...\",\n", + " 'common.resource.annotations',\n", + " 'm.09ywm7l'],\n", + " ['m.04kcrh0', 'tv.regular_tv_appearance.seasons', 'Fringe - Season 5'],\n", + " ['m.0yt7n52', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['m.0j13zhf',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Him/Herself'],\n", + " ['Dominic Monaghan', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['Sleepy Hollow', 'tv.tv_program.episodes', 'The Golem'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065hbf'],\n", + " ['John Noble', 'tv.tv_actor.starring_roles', 'm.0h5jyx4'],\n", + " ['Person', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Ian Holm', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.010h3gm8', 'film.performance.character', 'Pat Shepherd'],\n", + " ['m.09y0jjp',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Reginald Burchell'],\n", + " ['Liv Tyler', 'people.person.profession', 'Model'],\n", + " ['Karl Urban', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Ian Holm', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['Samantha Noble', 'people.person.place_of_birth', 'Adelaide'],\n", + " ['Country of nationality', 'rdf-schema#domain', 'Person'],\n", + " ['m.0yq9dfg', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['The Substance of Fire', 'theater.theater_production.cast', 'm.010g8n89'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'Ian McKellen'],\n", + " ['m.0h5jykf', 'film.performance.film', 'The Last Airbender'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyhq'],\n", + " ['John Noble', 'film.actor.film', 'm.012zhxfq'],\n", + " ['m.0h5jykf', 'film.performance.character', 'The Dragon Spirit'],\n", + " ['Penny Noble',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Country of nationality'],\n", + " ['Henry Parrish', 'tv.tv_character.appeared_in_tv_episodes', 'm.011xd9v_'],\n", + " ['Actor', 'business.job_title.people_with_this_title', 'm.012yds03'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Ectodermal dysplasia'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011f61l_'],\n", + " ['Howard Peet', 'film.film_character.portrayed_in_films', 'm.02vd0pq'],\n", + " ['Miranda Otto', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['m.0h5jyhb', 'film.performance.actor', 'John Noble'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Parkinson's disease\"],\n", + " ['David Wenham', 'award.award_winner.awards_won', 'm.09k3pgy'],\n", + " ['m.0g0h_jy', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Penny Noble', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0107yftd'],\n", + " ['m.0h5jyjp', 'film.performance.actor', 'John Noble'],\n", + " ['Film actor', 'common.topic.article', 'm.01z0fs9'],\n", + " ['m.0h5jyl5', 'film.performance.film', 'Call Me Mr. Brown'],\n", + " ['Cate Blanchett', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065kdb'],\n", + " ['Isaac Geldhart', 'theater.theater_character.portrayed_by', 'm.010g8n89'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Ian Holm'],\n", + " ['Andy Serkis',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Screen Actors Guild',\n", + " 'award.award_presenting_organization.award_categories_presented',\n", + " 'Screen Actors Guild Award for Outstanding Performance by a Cast in a Motion Picture'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jygz'],\n", + " ['m.09w22ld',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Karl Urban'],\n", + " ['m.0h5jyw6', 'tv.regular_tv_appearance.character', 'Anatoly Markov'],\n", + " ['m.011xd9wf', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Actor',\n", + " 'award.award_discipline.awards_in_this_discipline',\n", + " 'Golden Globe Award for Best Actress – Motion Picture – Musical or Comedy'],\n", + " ['Film actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.011skjfy'],\n", + " ['The Lord of the Rings: The Return of the King',\n", + " 'film.film.prequel',\n", + " 'The Lord of the Rings: The Two Towers'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Burn'],\n", + " ['m.0w2y671',\n", + " 'film.personal_film_appearance.film',\n", + " 'Ringers: Lord of the Fans'],\n", + " ['m.09p3lzv', 'tv.tv_guest_role.character', 'Dr Madsen'],\n", + " ['Samantha Noble', 'film.actor.film', 'm.0s95k3p'],\n", + " ['Jess Noble', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['Theatre Director', 'people.profession.specializations', 'Opera Director'],\n", + " ['Jess Noble', 'common.topic.notable_types', 'Person'],\n", + " ['m.012yds03', 'business.employment_tenure.title', 'Actor'],\n", + " ['Orlando Bloom', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.0h5jygz', 'film.performance.actor', 'John Noble'],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.risk_factors',\n", + " 'heart attack'],\n", + " ['John Noble', 'common.topic.webpage', 'm.09w6tcg'],\n", + " ['Daniel Noble', 'freebase.valuenotation.has_value', 'Place of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0_x50xn'],\n", + " ['John Noble', 'common.topic.notable_types', 'Film actor'],\n", + " ['John Noble', 'tv.tv_actor.guest_roles', 'm.09p3lzv'],\n", + " ['Viggo Mortensen', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.0n7xsws', 'award.award_honor.award_winner', 'Hugo Weaving'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Tetralogy of Fallot'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02zd4tr'],\n", + " ['Adelaide', 'location.location.containedby', 'Australia'],\n", + " ['Port Pirie', 'location.location.people_born_here', 'Andrew Rogers'],\n", + " ['Fringe - Season 3', 'tv.tv_series_season.series', 'Fringe'],\n", + " ['Bernard Hill', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Theatre Director',\n", + " 'education.field_of_study.students_majoring',\n", + " 'm.0_glgxt'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Stuttering'],\n", + " ['Stroke', 'medicine.risk_factor.diseases', 'Abdominal aortic aneurysm'],\n", + " ['m.09k3q0p', 'award.award_nomination.award_nominee', 'David Wenham'],\n", + " ['Film actor', 'base.descriptive_names.names.descriptive_name', 'm.0_zf_w6'],\n", + " ['m.011xd9v9', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['m.09p3lyp',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Day 6: 5:00 P.M. - 6:00 P.M.'],\n", + " ['Maria Pitillo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Fringe - Season 3', 'tv.tv_series_season.episodes', 'The Abducted'],\n", + " ['David Wenham', 'people.person.gender', 'Male'],\n", + " ['Film actor', 'freebase.documented_object.documentation', 'm.01z0fs9'],\n", + " ['John Noble', 'common.topic.image', 'Johnnoble 2003'],\n", + " ['m.0gw6dxs',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['Karl Urban', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.0kfyrvl',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['John Rhys-Davies', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0107yfxs'],\n", + " ['Jess Noble', 'people.person.profession', 'Model'],\n", + " ['m.05bvjyj',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['Voice Actor', 'common.topic.webpage', 'm.09y0jjp'],\n", + " ['Fringe - Season 1', 'tv.tv_series_season.regular_cast', 'm.04kcrh0'],\n", + " ['m.0h5jyl5', 'film.performance.actor', 'John Noble'],\n", + " ['Johnnoble 2003', 'common.image.size', 'm.029ns_t'],\n", + " [\"'Fringe': What did you think of J.J. Abrams' latest?\",\n", + " 'common.resource.annotations',\n", + " 'm.09y5k14'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0svpt1y'],\n", + " ['m.0b6zl9d', 'award.award_nomination.nominated_for', 'Fringe'],\n", + " ['Location', 'freebase.type_profile.published', 'Published'],\n", + " ['m.09p3lzv', 'tv.tv_guest_role.episodes_appeared_in', 'A Little Magic'],\n", + " ['Male', 'base.schemastaging.context_name.pronunciation', 'm.011sf4j6'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.011_yx1k'],\n", + " ['Port Pirie',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Port Pirie Regional Art Gallery'],\n", + " ['Kidney cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Viggo Mortensen', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.011qs93_',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyjp'],\n", + " ['Billy Boyd', 'award.award_winner.awards_won', 'm.0n7xsws'],\n", + " ['m.04mmfr8',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kawasaki disease'],\n", + " ['Denethor II', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Cardiovascular disease',\n", + " 'medicine.disease.parent_disease',\n", + " 'Cardiovascular disease'],\n", + " ['Liv Tyler',\n", + " 'medicine.notable_person_with_medical_condition.condition',\n", + " 'Attention deficit hyperactivity disorder'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0105b1n7'],\n", + " ['Maria Pitillo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Date of birth', 'type.property.expected_type', 'Date/Time'],\n", + " ['Voice Actor', 'common.topic.notable_for', 'g.1255_n54d'],\n", + " ['Fringe', 'tv.tv_program.regular_cast', 'm.04kcrh0'],\n", + " ['m.09k3pgy', 'award.award_honor.ceremony', \"9th Critics' Choice Awards\"],\n", + " ['m.09ycb67', 'common.webpage.topic', 'John Noble'],\n", + " ['m.0h5jywd', 'tv.regular_tv_appearance.series', 'Journeyman'],\n", + " ['Actor',\n", + " 'music.special_music_video_performance_type.special_music_video_performances',\n", + " 'm.0znpr6h'],\n", + " ['David Wenham', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.risk_factors',\n", + " 'Transient ischemic attack'],\n", + " ['m.04j647l',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Voice Actor',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01065h18'],\n", + " ['Hugo Weaving', 'award.award_nominee.award_nominations', 'm.0b4d5rz'],\n", + " ['Actor', 'common.topic.subjects', 'Randy M. Roberts'],\n", + " ['John Noble', 'tv.tv_actor.starring_roles', 'g.11by0r5fzq'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.0gxvt_4'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0105ysx4'],\n", + " ['m.010g8n89', 'theater.theater_role.actor', 'John Noble'],\n", + " ['Actor', 'base.lightweight.profession.professions_similar', 'Model'],\n", + " ['m.09p3lyp', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Ian Holm', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.09k3q0p',\n", + " 'award.award_nomination.award',\n", + " \"Critics' Choice Movie Award for Best Acting Ensemble\"],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0109n_4j'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Cate Blanchett'],\n", + " ['Country', 'freebase.type_hints.included_types', 'Location'],\n", + " ['m.0y8fm6n', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['David Wenham', 'people.person.profession', 'Voice Actor'],\n", + " ['Karl Urban', 'common.topic.notable_types', 'Film actor'],\n", + " ['Orlando Bloom', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0105yptg'],\n", + " ['David Wenham', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.011xd9v_', 'tv.tv_guest_role.episodes_appeared_in', 'The Golem'],\n", + " ['m.0j13zg1',\n", + " 'music.music_video_performance.special_music_video_performance_type',\n", + " 'Actor'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02_1x1x'],\n", + " ['m.0h5jxy8', 'people.marriage.type_of_union', 'Marriage'],\n", + " ['Film actor', 'type.type.expected_by', 'Film Actors'],\n", + " ['To', 'rdf-schema#range', 'Date/Time'],\n", + " ['m.09y5v2l',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j647l'],\n", + " ['Children', 'type.property.master_property', 'Parents'],\n", + " ['m.0gy5czt', 'tv.tv_guest_role.episodes_appeared_in', 'The Bishop Revival'],\n", + " ['Narrator', 'people.profession.specialization_of', 'Actor'],\n", + " ['Ian McKellen', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Pancreatic cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['John Noble', 'film.actor.film', 'm.0h5jyhb'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.011jy102'],\n", + " ['Port Pirie', 'common.topic.image', 'Port Pirie1'],\n", + " ['John Noble by Gage Skidmore',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'John Noble'],\n", + " ['m.0gd11j5', 'tv.tv_guest_role.episodes_appeared_in', 'Beech on the Run'],\n", + " ['Port Pirie', 'location.location.geolocation', 'm.02_lxz1'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.010133dr'],\n", + " ['m.09k3pgy', 'award.award_honor.award_winner', 'Sean Astin'],\n", + " ['General Booth', 'film.film_character.portrayed_in_films', 'm.0h5jygl'],\n", + " ['m.0yv3vs2', 'film.film_crew_gig.film_crew_role', 'Voice Actor'],\n", + " ['Film actor', 'type.type.expected_by', 'Actor'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvk3z'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.0kfyrv7'],\n", + " ['m.09y5k14',\n", + " 'common.webpage.resource',\n", + " \"'Fringe': What did you think of J.J. Abrams' latest?\"],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.0101320d'],\n", + " ['m.0s95k3p', 'film.performance.actor', 'Samantha Noble'],\n", + " ['Country of nationality', 'type.property.schema', 'Person'],\n", + " ['Liv Tyler',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Cate Blanchett', 'people.person.profession', 'Voice Actor'],\n", + " ['Actor', 'base.descriptive_names.names.descriptive_name', 'm.01012_9f'],\n", + " ['Sean Astin',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Film actor', 'type.type.expected_by', 'Actor'],\n", + " ['m.0b6zlcc', 'award.award_nomination.nominated_for', 'Fringe'],\n", + " ['Film actor', 'type.type.properties', 'Film dubbing performances'],\n", + " ['Samantha Noble', 'people.person.nationality', 'Australia'],\n", + " ['m.0b4d5rz', 'award.award_nomination.award_nominee', 'Karl Urban'],\n", + " ['The C.E.O.', 'tv.tv_character.appeared_in_tv_episodes', 'm.09p3l_l'],\n", + " ['m.0gd11j5', 'tv.tv_guest_role.actor', 'John Noble'],\n", + " ['Sean Astin', 'award.award_nominee.award_nominations', 'm.09k3q0p'],\n", + " ['m.0gxvv0h',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Film actor', 'type.type.properties', 'NY Times ID'],\n", + " ['m.0g0h_jy',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Day 6: 6:00 P.M. - 7:00 P.M.'],\n", + " ...],\n", + " [['m.02_1x1x',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Andrew Carnegie Mansion', 'location.location.containedby', 'Manhattan'],\n", + " ['The Stand', 'film.film.country', 'United States of America'],\n", + " ['New York City',\n", + " 'exhibitions.exhibition_subject.exhibitions_created_about_this_subject',\n", + " 'State of the Art: New York'],\n", + " ['Joakim Noah', 'people.person.gender', 'Male'],\n", + " ['Forward', 'sports.sports_position.sport', 'Basketball'],\n", + " ['New York City', 'book.book_subject.works', 'Let the Great World Spin'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnmg'],\n", + " ['Guerrilla Decontextualization and the 2012 Presidential Election Campaign (Part 1)',\n", + " 'book.written_work.school_or_movement',\n", + " 'African American'],\n", + " ['New York City',\n", + " 'visual_art.art_subject.artwork_on_the_subject',\n", + " 'Flags on Fifty-Seventh Street'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'National Anti-Slavery Standard'],\n", + " [\"CCNY Beavers men's basketball\",\n", + " 'sports.sports_team.location',\n", + " 'New York City'],\n", + " ['m.051qt5y',\n", + " 'travel.travel_destination_monthly_climate.travel_destination',\n", + " 'New York City'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0drxd0b'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnrb'],\n", + " ['m.0j13lxs', 'people.marriage.spouse', 'Yannick Noah'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Transient ischemic attack'],\n", + " ['Atlantic Theater Company',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['m.0dzbw6f', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Uppsala County', 'location.location.containedby', 'Sweden'],\n", + " ['m.010n_5cp', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Louis Armstrong House'],\n", + " ['m.0qjn3wq', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gout'],\n", + " ['m.010n_3n7', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Barrett's esophagus\"],\n", + " ['Kansas',\n", + " 'fictional_universe.fictional_setting.universe',\n", + " 'The Sacred Band of Stepsons universe'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5d1'],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.parent_disease',\n", + " 'Cardiovascular disease'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Prostate cancer'],\n", + " ['13 Going on 30', 'film.film.country', 'United States of America'],\n", + " ['Yannick Noah', 'music.lyricist.lyrics_written', 'Night of Blues'],\n", + " ['African American', 'common.topic.webpage', 'm.09w10n8'],\n", + " ['m.0519ccq',\n", + " 'travel.transportation.transport_terminus',\n", + " 'Newark Liberty International Airport'],\n", + " ['m.010n_5gv', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_5vc'],\n", + " ['Daily News', 'book.newspaper.circulation_areas', 'Manhattan'],\n", + " ['Statue of Liberty',\n", + " 'base.disneyana.disney_product_theme.disney_products_based_on_this_theme',\n", + " 'New York City Pooh'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.subsumes',\n", + " 'United States of America'],\n", + " ['Little Italy', 'location.location.containedby', 'Manhattan'],\n", + " ['m.0107blsl',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['m.0107bljp', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Americada'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Crocheron Park'],\n", + " ['New York City', 'base.biblioness.bibs_topic.subsumes', 'New York City'],\n", + " ['The Stand', 'film.film.genre', 'Drama'],\n", + " ['Manhattan', 'location.location.containedby', 'New York City'],\n", + " ['Sweden', 'location.country.first_level_divisions', 'Gävleborg County'],\n", + " ['John Reese', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['New York City', 'book.book_subject.works', 'What Happened to Anna K.'],\n", + " ['Time', 'music.composition.language', 'English Language'],\n", + " ['Sweden', 'location.country.administrative_divisions', 'Norrbotten County'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'World Voices Festival Celebrates Literary Diplomacy (part 1 of 2)'],\n", + " ['Center', 'common.topic.notable_types', 'Basketball Position'],\n", + " ['Athlete', 'common.topic.notable_types', 'Profession'],\n", + " ['Epilepsy', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Viceroy New York', 'location.location.containedby', 'New York'],\n", + " ['Central Park',\n", + " 'book.book_subject.works',\n", + " 'Passing Through Eden: Photographs of Central Park'],\n", + " ['Lemurian windows into any place or time',\n", + " 'fictional_universe.fictional_object.featured_in_fictional_universe',\n", + " 'Tempus Unbound'],\n", + " ['New York City', 'book.book_subject.works', 'Lush Life: A Novel'],\n", + " ['heart attack',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['What Happened to Anna K.', 'book.written_work.subjects', 'New York City'],\n", + " ['Guerrilla Decontextualization and the 2012 Presidential Election Campaign (Part 1)',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.07n73w_'],\n", + " ['Basketball player',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010h540p'],\n", + " ['“In addition to her musical sensibilities, [Nina Simone] also possessed a social/political consciousness that was magnified by friendships with individuals like the playwright Lorraine Hansberry, author James Baldwin, and author Langston Hughes, all of whom at various times lent their artistry to the struggle for racial equality.”',\n", + " 'media_common.quotation.subjects',\n", + " 'African American'],\n", + " ['Sweden',\n", + " 'location.country.administrative_divisions',\n", + " 'Västerbotten County'],\n", + " ['Times Square',\n", + " 'book.book_subject.works',\n", + " 'Times Square Spectacular: Lighting up Broadway'],\n", + " ['m.02wk9_t', 'location.adjoining_relationship.adjoins', 'New York City'],\n", + " ['m.0qh0czg', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0j7v__g'],\n", + " ['Southern United States',\n", + " 'book.book_subject.works',\n", + " 'Harlem Renaissance Way Down South'],\n", + " ['m.010n_4vd',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'New York City'],\n", + " [\"Parkinson's disease\",\n", + " 'medicine.disease.symptoms',\n", + " 'Seborrheic dermatitis'],\n", + " ['Västerbotten County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['Statue of Liberty National Monument',\n", + " 'location.location.containedby',\n", + " 'New York'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0k6kggv'],\n", + " ['m.02_98hd',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['New York City',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '\\\\\"History dressed up in the glow of love’s kiss turned grief into beauty.\\\\\"'],\n", + " ['New York City Department of Housing Preservation and Development',\n", + " 'government.governmental_body.members',\n", + " 'm.04j7jcy'],\n", + " ['John F. Kennedy International Airport',\n", + " 'aviation.airport.serves',\n", + " 'Viceroy New York'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0jt07bd'],\n", + " ['Claire Bennet', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Sweden', 'location.country.administrative_divisions', 'Örebro County'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Village Voice'],\n", + " ['m.010n_6ff',\n", + " 'film.film_regional_release_date.film_regional_debut_venue',\n", + " '2013 Vienna International Film Festival'],\n", + " ['Central Park', 'location.location.containedby', 'Manhattan'],\n", + " ['Gotland County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['New York City', 'base.popstra.location.vacationers', 'm.064sw_s'],\n", + " ['Kalmar County',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'Sweden'],\n", + " [\"Florida Gators men's basketball\",\n", + " 'sports.sports_team.sport',\n", + " 'Basketball'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1hdscp'],\n", + " ['Midnight Cowboy', 'film.film.genre', 'Drama'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnm8'],\n", + " ['Manhattan',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The New York Times'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5fm'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'UAE Healthy Kidney 10K'],\n", + " ['Manhattan',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Wall Street Journal'],\n", + " ['Male', 'common.topic.article', 'm.05zpq8'],\n", + " ['New York City',\n", + " 'location.location.nearby_airports',\n", + " 'East 34th Street Heliport'],\n", + " ['MBSF - Private Jets', 'location.location.containedby', 'New York'],\n", + " ['New York City', 'tv.tv_location.tv_shows_filmed_here', 'The Stand'],\n", + " ['Kalmar County', 'location.administrative_division.country', 'Sweden'],\n", + " ['Freie Arbeiter Stimme', 'book.newspaper.circulation_areas', 'New York'],\n", + " ['Atlantic Theater Company',\n", + " 'education.educational_institution.subsidiary_or_constituent_schools',\n", + " 'Atlantic Theater Company'],\n", + " ['Gävleborg County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['New York City', 'common.topic.subject_of', 'Engage the Crowd NYC'],\n", + " ['New York City',\n", + " 'base.militaryinfiction.location_in_fiction.contained_by',\n", + " 'Lemurian windows into any place or time'],\n", + " ['Harlem Renaissance',\n", + " 'visual_art.art_period_movement.associated_artworks',\n", + " 'Songs of My People'],\n", + " ['Staten Island', 'location.location.containedby', 'New York'],\n", + " ['New York City', 'film.film_subject.films', 'The Warriors'],\n", + " ['Morgen Freiheit', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11b7tm_xvm'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Dry Manhattan: Prohibition in New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'Traffic Violations Bureau'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0qjn3sx'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Morgen Freiheit'],\n", + " ['A.I.R. Gallery',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Sun'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Norrbotten County'],\n", + " ['Second-Hand NY',\n", + " 'base.webvideo.internet_video_series.genre',\n", + " 'New York City'],\n", + " ['m.0j2gdp4', 'base.schemastaging.athlete_salary.athlete', 'Joakim Noah'],\n", + " ['Lenny Cooke', 'film.film.produced_by', 'Adam Shopkorn'],\n", + " ['Lenny Cooke', 'film.film.other_crew', 'm.010p01ry'],\n", + " ['Times Square', 'location.location.containedby', 'New York City'],\n", + " ['Tennis Player', 'freebase.type_hints.included_types', 'Person'],\n", + " ['Chinatown', 'location.location.containedby', 'New York'],\n", + " ['2013 Tribeca Film Festival', 'time.event.locations', 'New York City'],\n", + " ['African American', 'common.topic.webpage', 'm.09ym_fk'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.010h53v9'],\n", + " ['Catholic Worker', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_3b_'],\n", + " ['m.010_xz6_', 'award.award_nomination.ceremony', '2014 ESPY Awards'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_515'],\n", + " ['New York Communist', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Sweden', 'location.country.administrative_divisions', 'Blekinge County'],\n", + " ['Wizard Wars',\n", + " 'base.militaryinfiction.event_in_fiction.location',\n", + " 'New York City'],\n", + " ['American Folk Art Museum',\n", + " 'location.location.containedby',\n", + " 'New York City'],\n", + " ['m.051qt6s',\n", + " 'travel.travel_destination_monthly_climate.travel_destination',\n", + " 'New York City'],\n", + " ['New York',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'New York City'],\n", + " ['Countdown of 10 Great Moments in 2010 Part 2 Beyonce and Jay-Z',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['City Hall', 'common.topic.notable_types', 'Film'],\n", + " ['New York City Fire Department',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['New York City', 'book.book_subject.works', 'A Christmas Caroline'],\n", + " ['Yannick Noah', 'common.topic.image', 'Yannick Noa POPB Octobre 2004 - 01'],\n", + " ['The London 2012 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney Stone'],\n", + " ['Yannick Noah', 'music.artist.album', 'g.119pgj22y'],\n", + " ['New York Star', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['New York City',\n", + " 'location.location.primarily_containedby',\n", + " 'Area code 347'],\n", + " ['Memory-Song Painted Gold: for the Blue Yusef Lateef (1920-2013)',\n", + " 'common.topic.subjects',\n", + " 'African American'],\n", + " ['New York Call', 'book.periodical.language', 'American English'],\n", + " ['Brooklyn Botanic Garden',\n", + " 'location.location.containedby',\n", + " 'New York City'],\n", + " ['Spanish Language', 'language.human_language.region', 'Europe'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Freie Arbeiter Stimme'],\n", + " ['Central Park in the Dark: More Mysteries of Urban Wildlife',\n", + " 'book.written_work.subjects',\n", + " 'Central Park'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d521lm'],\n", + " ['m.0x2x5q1',\n", + " 'government.government_position_held.governmental_body',\n", + " 'New York City Council'],\n", + " ['New York City', 'travel.travel_destination.climate', 'm.051qt3z'],\n", + " ['Daily Worker', 'book.newspaper.circulation_areas', 'New York'],\n", + " ['Lenny Cooke', 'film.film.personal_appearances', 'm.0107blx6'],\n", + " ['Yannick Noah', 'music.lyricist.lyrics_written', 'Yola'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'The Cloisters'],\n", + " ['Joakim Noah', 'people.person.profession', 'Basketball player'],\n", + " ['m.0g8qg3r', 'base.firsts.first_achievement.category', 'African American'],\n", + " ['Ethnicity', 'freebase.type_profile.published', 'Published'],\n", + " ['Lemurian windows into any place or time',\n", + " 'base.militaryinfiction.location_in_fiction.setting_type',\n", + " 'Lemuria'],\n", + " ['Tennis Player', 'freebase.type_profile.equivalent_topic', 'Tennis player'],\n", + " ['Basketball player',\n", + " 'people.profession.corresponding_type',\n", + " 'Basketball Player'],\n", + " ['African American',\n", + " 'visual_art.art_subject.artwork_on_the_subject',\n", + " 'Bright Skylark Literary Productions'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Savannah: Brokers, Bankers, and Bay Lane'],\n", + " ['Joakim Noah', 'common.topic.image', 'Joakim Noah3'],\n", + " ['Joakim Noah', 'award.award_winner.awards_won', 'm.010brmqg'],\n", + " ['m.0b48pfv', 'common.webpage.topic', 'African American'],\n", + " ['m.0w5t5k0', 'sports.pro_sports_played.athlete', 'Yannick Noah'],\n", + " ['Västernorrland County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['Central Park in the Dark: More Mysteries of Urban Wildlife',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Östergötland County'],\n", + " ['Midnight Cowboy', 'film.film.language', 'English Language'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['Bright Skylark Literary Productions',\n", + " 'visual_art.artwork.art_subject',\n", + " 'African American'],\n", + " ['Il Progresso Italo-Americano',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0dy5t4b'],\n", + " ['Manhattan',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States of America'],\n", + " ['African American', 'common.topic.webpage', 'm.09xsqkm'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The New York Blade'],\n", + " ['Sweden',\n", + " 'location.country.first_level_divisions',\n", + " 'Västernorrland County'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'The Battle for New York: The City at the Heart of the American Revolution'],\n", + " [\"Report on 2011 International Year part 4 Haiti's Hope Now and Tomorrow\",\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Richie'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5db'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Nowy Dziennik'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['New York City', 'wine.wine_region.wines', 'Ice wine'],\n", + " ['African American',\n", + " 'base.firsts.first_achievement_category.firsts',\n", + " 'm.0g8qg34'],\n", + " ['m.0j2hqb0', 'people.sibling_relationship.sibling', 'Yélena Noah'],\n", + " ['Black History Month Enhanced by International Year for People of African Descent',\n", + " 'book.written_work.subjects',\n", + " 'Black History Month'],\n", + " ['Cime', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['New York City',\n", + " 'base.jewlib.parent_institution.judaica_owning_units',\n", + " 'New York Public Library Main Branch'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Solomon R. Guggenheim Museum'],\n", + " ['Museum of Sex',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['United States of America',\n", + " 'location.location.primarily_containedby',\n", + " 'North America'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Statue of Liberty'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1g8hdx'],\n", + " ['Madison Square Garden',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0qfg_pk'],\n", + " ['m.09vqf06',\n", + " 'tennis.tennis_tournament_championship.winner',\n", + " 'Yannick Noah'],\n", + " ['Basketball Player', 'type.type.properties', 'Position(s)'],\n", + " ['m.05gmrc3', 'base.firsts.first_achievement.category', 'African American'],\n", + " ['Joakim Noah', 'award.award_winner.awards_won', 'm.010q9zv3'],\n", + " ['African American',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '“However the world community chooses to interpret it, where Michael Jackson himself is concerned, if the purpose of a legacy is to help make the world a richer, more fulfilling, and more humane place than it was during one’s lifetime, he can rest in peace knowing he did exactly that.” --from Michael Jackson, Legacies of a Globetrotting Moonwalking Philanthropist'],\n", + " ['Crocheron Park',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Notebook on Black History Month 2012 (Part 1): Carter G. Woodson and Company',\n", + " 'book.written_work.school_or_movement',\n", + " 'African American'],\n", + " ['Times Square', 'film.film_location.featured_in_films', 'Midnight Cowboy'],\n", + " ['France',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'New York City Council'],\n", + " ['Bruce Wayne',\n", + " 'comic_books.comic_book_character.story_specific_appearances',\n", + " 'A Death in the Family, Part Four'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Dupuytren's contracture\"],\n", + " ['The New Black',\n", + " 'book.written_work.school_or_movement',\n", + " 'Harlem Renaissance'],\n", + " ['African Voices A Soulful Collection of Art and Literature',\n", + " 'book.periodical.subjects',\n", + " 'Harlem Renaissance'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'g.121764rk'],\n", + " ['African American',\n", + " 'visual_art.art_subject.artwork_on_the_subject',\n", + " 'African-American Monument'],\n", + " ['African American',\n", + " 'people.ethnicity.languages_spoken',\n", + " 'Sea Island Creole English Language'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Actinic keratosis'],\n", + " ['The New York Times', 'book.newspaper.circulation_areas', 'Staten Island'],\n", + " ['New York City', 'travel.travel_destination.climate', 'm.051qt47'],\n", + " ['New York City', 'location.location.containedby', 'New York'],\n", + " ['Josh Safdie', 'people.person.nationality', 'United States of America'],\n", + " ['Stevie Nichols',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'New York City'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0rjhs7z'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.04j7jcy'],\n", + " ['New York',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'United States Bankruptcy Court for the Southern District of New York'],\n", + " ['m.0_z47jj',\n", + " 'tennis.tennis_tournament_championship.winner',\n", + " 'Yannick Noah'],\n", + " ['New York City', 'base.popstra.location.vacationers', 'm.064sltb'],\n", + " ['m.0v3pbcl', 'music.group_membership.member', 'Yannick Noah'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Mason Greyback'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'Sesame Street'],\n", + " ['New York City', 'location.location.adjoin_s', 'm.02wk9_t'],\n", + " ['Daily News', 'book.newspaper.circulation_areas', 'Staten Island'],\n", + " ['Let the Great World Spin', 'book.written_work.subjects', 'New York City'],\n", + " ['m.09y422x', 'common.webpage.topic', 'African American'],\n", + " ['Rockefeller Center',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['Ishi Nakamura',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'New York City'],\n", + " ['Örebro County', 'location.administrative_division.country', 'Sweden'],\n", + " ['m.05blxxj',\n", + " 'sports.sports_award.season',\n", + " \"2006 NCAA Men's Division I Basketball Tournament\"],\n", + " ['2013–14 NBA season', 'sports.sports_league_season.awards', 'm.010c7kcl'],\n", + " ['Yannick Noah', 'music.artist.album', 'Pokhara'],\n", + " ['m.010n_6ff', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Harlem Renaissance Way Down South',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['m.010brmqg', 'freebase.valuenotation.has_no_value', 'Ceremony'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Notebook on Black History Month 2012 (Part 1): Carter G. Woodson and Company'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'East Village Other'],\n", + " ['Västernorrland County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Staten Island Ferry'],\n", + " ['Metropolitan Museum of Art',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['Wizard Wars',\n", + " 'fictional_universe.event_in_fiction.includes_events',\n", + " 'Askelon follows Cime to the 20th Century'],\n", + " ['Japan Society of New York',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['m.09ys0vx', 'common.webpage.topic', 'African American'],\n", + " ['Lemuria',\n", + " 'fictional_universe.type_of_fictional_setting.settings',\n", + " 'Lemuria'],\n", + " ['Southern United States',\n", + " 'book.book_subject.works',\n", + " 'Savannah: Brokers, Bankers, and Bay Lane'],\n", + " ['Marymount Theatre School',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['Basketball player',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01069wsq'],\n", + " ['Pancreatic cancer',\n", + " 'medicine.icd_9_cm_classification.parent_classification',\n", + " 'Pancreatic cancer'],\n", + " ['City Hall', 'fictional_universe.work_of_fiction.setting', 'New York City'],\n", + " ['Flatiron Building', 'location.location.containedby', 'New York'],\n", + " ['Passing Through Eden: Photographs of Central Park',\n", + " 'book.written_work.subjects',\n", + " 'Central Park'],\n", + " ['Male',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Attention deficit hyperactivity disorder'],\n", + " ['Basketball player',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Derek Roman'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0nkjx50'],\n", + " ['Manhattan', 'location.us_county.county_seat', 'Manhattan'],\n", + " ['Shleppers Moving & Storage',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York'],\n", + " ['m.0vyrky6', 'music.group_membership.member', 'Yannick Noah'],\n", + " ['New York Public Library Main Branch',\n", + " 'location.location.containedby',\n", + " 'New York City'],\n", + " ['m.010n_3s7', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Exit Ghost', 'book.written_work.subjects', 'New York City'],\n", + " ['m.04j7jch',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'New York City'],\n", + " ['Crocheron Park', 'location.location.containedby', 'New York'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Der Yid'],\n", + " ['Basketball Player',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Athlete'],\n", + " ['Madison Square Garden',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['\\\\\"From her gospel-singing mother Cissy Houston, her legendary pop-diva cousin Dionne Warwick, and her Queen of Soul godmother Aretha Franklin, she [Whitney Houston] inherited gifts for skillfully interpreting lyrics and endowing them with new depth and jeweled nuance.\\\\\"',\n", + " 'media_common.quotation.subjects',\n", + " 'African American'],\n", + " ['Staten Island',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Daily News'],\n", + " ['Stockholm County', 'location.administrative_division.country', 'Sweden'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Midtown Manhattan as seen from the GE Building.',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'New York City'],\n", + " ['Sweden', 'location.country.first_level_divisions', 'Gotland County'],\n", + " ['Chrysler Building',\n", + " 'book.book_subject.works',\n", + " 'The Chrysler Building: Creating a New York Icon Day by Day'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'SEO Services New York'],\n", + " ['World Voices Festival Celebrates Literary Diplomacy (part 1 of 2)',\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['Radical Homosexual Agenda',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['New York Post', 'book.newspaper.circulation_areas', 'Staten Island'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Frick Collection'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Cultural Affairs'],\n", + " ['The Ungovernable City: John Lindsay and His Struggle to Save New York',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York Public Library for the Performing Arts',\n", + " 'location.location.containedby',\n", + " 'Manhattan'],\n", + " ['Lenny Cooke', 'film.film.other_crew', 'm.0wffpyb'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.0wffpj8'],\n", + " ['Mac Taylor', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'City Hall'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gastritis'],\n", + " ['The Cloisters',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0zvdpy9'],\n", + " ['The Wisdom of W.E.B. Du Bois',\n", + " 'book.written_work.subjects',\n", + " 'African diaspora'],\n", + " ['City Hall', 'film.film.language', 'English Language'],\n", + " ['Metropolitan Museum of Art', 'location.location.containedby', 'New York'],\n", + " ['New York City', 'film.film_subject.films', 'Midnight Cowboy'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Sebaceous cyst'],\n", + " ['m.09wtdky', 'common.webpage.topic', 'African American'],\n", + " ['African American', 'common.topic.webpage', 'm.09wwv2n'],\n", + " ['m.04j7j70',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'New York City'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['African American',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '\\\\\"The fact that a significant number of black youths\\' deaths can be described as the result of “black on black” violence does not negate the truth or root causes at the core of the issue.\\\\\"'],\n", + " ['Brooklyn Bridge', 'location.location.containedby', 'New York City'],\n", + " ['African American', 'common.topic.webpage', 'm.09wtdky'],\n", + " ['Dalarna County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['Area code 347',\n", + " 'location.location.contains_major_portion_of',\n", + " 'New York City'],\n", + " ['English Language',\n", + " 'language.human_language.dialects',\n", + " 'African American Vernacular English'],\n", + " ['Norrbotten County', 'location.administrative_division.country', 'Sweden'],\n", + " ['m.02nqxbv', 'sports.sports_league_draft_pick.team', 'Chicago Bulls'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department for the Aging'],\n", + " ['Joakim Noah', 'sports.drafted_athlete.drafted', 'm.02nqxbv'],\n", + " ['Notebook on Black History Month 2012 (Part 1): Carter G. Woodson and Company',\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['m.010n_7lj', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['New York City',\n", + " 'travel.travel_destination.local_transportation',\n", + " 'New York City Subway'],\n", + " ['m.010q9zv3', 'freebase.valuenotation.has_no_value', 'Ceremony'],\n", + " ['Der Groyser Kundes', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Basketball player', 'people.profession.specialization_of', 'Athlete'],\n", + " ['92nd Street Y',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['Kansas',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['New York City',\n", + " 'location.hud_foreclosure_area.total_90_day_vacant_residential_addresses',\n", + " 'm.07f62rx'],\n", + " ['Sullivan & Galleshaw, LLP',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Times Square'],\n", + " ['MBSF - Private Jets', 'organization.organization.locations', 'New York'],\n", + " ['m.064svkj', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Trayvon Martin, Robert Lee, and Millions of Tears Fallen (Part 1)'],\n", + " ['Exit Ghost', 'book.written_work.original_language', 'English Language'],\n", + " ['Brooklyn Botanic Garden', 'location.location.containedby', 'New York'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'New York Perl Mongers'],\n", + " ['Madison Square Garden',\n", + " 'sports.sports_facility.teams',\n", + " 'New York Rangers'],\n", + " ['Trial Heat', 'tv.tv_program.filming_locations', 'New York City'],\n", + " ['African American',\n", + " 'book.periodical_subject.periodicals',\n", + " 'Savannah Tribune'],\n", + " ['m.03p7z1w', 'education.education.student', 'Joakim Noah'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02_98hd'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0dzr8zz'],\n", + " ['Donald Flack, Jr.',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Male'],\n", + " ['Tempus Unbound',\n", + " 'fictional_universe.fictional_universe.languages',\n", + " 'English Language'],\n", + " ['Benny Safdie', 'people.person.place_of_birth', 'New York'],\n", + " ['Female', 'common.topic.subjects', 'ZasPorn'],\n", + " ['African American',\n", + " 'base.firsts.first_achievement_category.firsts',\n", + " 'm.0g8qfbh'],\n", + " ['The Morning Star', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['New York City', 'base.popstra.location.vacationers', 'm.064smxq'],\n", + " ['Basketball Player', 'freebase.type_profile.kind', 'Title'],\n", + " ['Nighthawks', 'visual_art.artwork.art_subject', 'New York City'],\n", + " ['Lemurian windows into any place or time',\n", + " 'fictional_universe.fictional_object.featured_in_fictional_universe',\n", + " 'The Sacred Band of Stepsons universe'],\n", + " ['New York Post', 'book.periodical.language', 'English Language'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1jnp8n'],\n", + " ['Yannick Noah', 'music.lyricist.lyrics_written', 'My Love Is Gone'],\n", + " ['m.0qc8kkj', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['m.0j2hqb0', 'people.sibling_relationship.sibling', 'Joakim Noah'],\n", + " ['Lenny Cooke', 'film.film.country', 'United States of America'],\n", + " ['m.064sr1t', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['m.0649tj0', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['The Wisdom of W.E.B. Du Bois',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['Yannick Noah', 'common.topic.image', 'Hélène Ségara with Yannick Noah'],\n", + " ['New York City Council',\n", + " 'government.governmental_body.jurisdiction',\n", + " 'New York City'],\n", + " ['Chrysler Building',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['The Warriors', 'film.film.featured_film_locations', 'New York City'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Village Voice'],\n", + " ['Songs of My People Art Exhibition Opens at Penn Center',\n", + " 'book.written_work.subjects',\n", + " 'African American'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0qb8t60'],\n", + " ['Statue of Liberty',\n", + " 'location.location.containedby',\n", + " 'Statue of Liberty National Monument'],\n", + " ['m.09y065m', 'common.webpage.topic', 'African American'],\n", + " ['\\\\\"From her gospel-singing mother Cissy Houston, her legendary pop-diva cousin Dionne Warwick, and her Queen of Soul godmother Aretha Franklin, she [Whitney Houston] inherited gifts for skillfully interpreting lyrics and endowing them with new depth and jeweled nuance.\\\\\"',\n", + " 'media_common.quotation.source',\n", + " 'Notebook on Black History Month 2012 (Part 6) The Consecrated Soul of Whitney Houston'],\n", + " ['The Wall Street Journal',\n", + " 'base.meedan.arabic_language_media_source.city',\n", + " 'New York City'],\n", + " ['Sweden', 'location.country.first_level_divisions', 'Västerbotten County'],\n", + " ['m.0rsqxq3', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Yannick Noah', 'base.schemastaging.athlete_extra.coaches', 'm.0j13vpp'],\n", + " ['m.010n_515', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0w9vg9x'],\n", + " ['New York City', 'location.statistical_region.population', 'g.1jmcbfml7'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d5218t'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Headless Horseman Hayrides'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Donald Flack, Jr.'],\n", + " ['Passing Through Eden: Photographs of Central Park',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['The Consecrated Soul of Whitney Houston',\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.04j7j70'],\n", + " ['Area code 917', 'location.location.contains', 'New York City'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Statue of Liberty National Monument'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of City Planning'],\n", + " ['m.0nkjwdb', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Basketball player',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01069wpd'],\n", + " ['Dana Barrett',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'New York Rises: Photographs by Eugene de Salignac'],\n", + " ['Sesame Street', 'tv.tv_program.languages', 'English Language'],\n", + " ['m.0gxvv0q',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Let the Great World Spin',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['Wave Hill', 'location.location.containedby', 'New York'],\n", + " ['Imagination Playground at Burling Slip',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Corneal abrasion'],\n", + " ['Yannick Noah', 'music.artist.album', 'Donne-moi une vie'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnxm'],\n", + " ['Brooklyn Botanic Garden',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['African American', 'common.topic.webpage', 'm.09ys4my'],\n", + " ['Gotland County', 'location.location.containedby', 'Sweden'],\n", + " ['The Stand', 'tv.tv_program.country_of_origin', 'United States of America'],\n", + " ['Kalmar County', 'location.location.containedby', 'Sweden'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64q8'],\n", + " ['m.02zd4tr',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['m.0b6vrkl',\n", + " 'education.education.institution',\n", + " 'Poly Prep Country Day School'],\n", + " [\"The Soul of the Soulless City ('New York - an Abstraction')\",\n", + " 'visual_art.artwork.art_subject',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'tv.tv_location.tv_episodes_filmed_here',\n", + " 'Old MacDonald Had a Curve'],\n", + " ['m.0j2gdqx',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Earth', 'base.locations.planets.countries_within', 'France'],\n", + " ['New York City',\n", + " 'base.arthist.helynevek.intezmeny',\n", + " 'Metropolitan Museum of Art'],\n", + " ['m.0dm7bqp', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Lenny Cooke', 'film.film.personal_appearances', 'm.0107blvs'],\n", + " ['m.0k8lwgq',\n", + " 'olympics.olympic_athlete_affiliation.olympics',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['Statue of Liberty National Monument',\n", + " 'location.location.contains',\n", + " 'Statue of Liberty'],\n", + " ['The Approaching 100th Anniversary of the Harlem Renaissance (part 2)',\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['Basketball Player', 'type.type.expected_by', 'Basketball Players'],\n", + " ['m.0fjw72l', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['m.0_yspgl',\n", + " 'tennis.tennis_tournament_championship.winner',\n", + " 'Yannick Noah'],\n", + " ['Harlem Renaissance',\n", + " 'exhibitions.exhibition_subject.exhibitions_created_about_this_subject',\n", + " 'Songs of My People'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnrq'],\n", + " ['African American', 'common.topic.webpage', 'm.09w7g5_'],\n", + " ['m.010n_4bd', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Chrysler Building', 'location.location.containedby', 'New York'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney cancer'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_7lj'],\n", + " ['Gothenburg', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['New York City', 'location.location.geolocation', 'm.0kfn9s'],\n", + " ['New York, Empire City: 1920-1945',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'A Death in the Family, Part Four'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnqs'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " \"Barron's\"],\n", + " ['France', 'base.locations.countries.planet', 'Earth'],\n", + " ['Yannick Noah', 'music.group_member.membership', 'm.0w38jlq'],\n", + " ['Yannick Noah',\n", + " 'tennis.tennis_tournament_champion.tennis_titles',\n", + " 'm.0_z1664'],\n", + " ['Freedomland U.S.A.', 'location.location.containedby', 'New York'],\n", + " ['m.09y4_xz', 'common.webpage.topic', 'African American'],\n", + " ['Yannick Noah', 'people.person.places_lived', 'm.03pp9zg'],\n", + " ['ELEMENTAL: The Power of Illuminated Love',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['Joakim Noah', 'people.person.place_of_birth', 'New York City'],\n", + " ['MBSF - Private Jets',\n", + " 'location.location.nearby_airports',\n", + " 'LaGuardia Airport'],\n", + " ['Basketball player',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Terrence Wall'],\n", + " ['African American', 'medicine.risk_factor.diseases', 'Pancreatic cancer'],\n", + " ['New York City', 'book.book_subject.works', 'Paradise Travel'],\n", + " ['Athlete', 'people.profession.specializations', 'Basketball player'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Tesla Science Center at Wardenclyffe'],\n", + " ['North America',\n", + " 'location.location.contains_major_portion_of',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Kansas'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'New York World'],\n", + " ['Rockefeller Center', 'location.location.containedby', 'New York'],\n", + " ['New York Call', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Basketball Player', 'freebase.type_hints.included_types', 'Athlete'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Transportation'],\n", + " ['Joakim Noah', 'people.person.education', 'm.03p7z1w'],\n", + " ['New York City', 'base.petbreeds.city_with_dogs.top_breeds', 'm.063sbhn'],\n", + " ['New York City',\n", + " 'common.topic.image',\n", + " 'Midtown Manhattan as seen from the GE Building.'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Probation'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.0107blfp'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'American Museum of Natural History'],\n", + " ['Manhattan',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'New York Press'],\n", + " ['Europe', 'base.locations.continents.countries_within', 'Sweden'],\n", + " ['African American', 'medicine.risk_factor.diseases', 'Asthma'],\n", + " ['heart attack', 'medicine.disease.risk_factors', 'Male'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Investigation'],\n", + " ['New York City', 'base.popstra.business_location.customer', 'm.06527_0'],\n", + " ['New York City',\n", + " 'sports.sports_team_location.teams',\n", + " 'New York Golden Blades'],\n", + " ['John F. Kennedy International Airport',\n", + " 'travel.transport_terminus.travel_destinations_served',\n", + " 'm.0519ccq'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Elite SEM'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Savannah Talks Troy Anthony Davis No. 12: U.S. Supreme Court Rejects Appeal'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tour_operators',\n", + " 'Adventures by Disney'],\n", + " ['Harlem Renaissance Way Down South',\n", + " 'common.topic.subjects',\n", + " 'African American'],\n", + " ['New York City',\n", + " 'base.jewlib.parent_institution.judaica_owning_units',\n", + " 'Schomburg Center for Research in Black Culture'],\n", + " ['Triumph Hotels',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['m.010qt03r', 'freebase.valuenotation.has_no_value', 'Ceremony'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Urolithiasis'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Museum of Arts and Design'],\n", + " ['Bronx Beat', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Phone Trick',\n", + " 'base.thoroughbredracing.thoroughbred_racehorse.sex',\n", + " 'Male'],\n", + " ['Cime',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " 'Lemurian windows into any place or time'],\n", + " ['New York Rises: Photographs by Eugene de Salignac',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['m.065pd1c', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['m.0w06qbc', 'music.group_membership.member', 'Yannick Noah'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " '10 Great Moments in African-American History'],\n", + " ['LaGuardia Airport',\n", + " 'travel.transport_terminus.travel_destinations_served',\n", + " 'm.0519ccq'],\n", + " ['m.09w10n8', 'common.webpage.topic', 'African American'],\n", + " ['Manhattan',\n", + " 'location.place_with_neighborhoods.neighborhoods',\n", + " 'Little Italy'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnhz'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5fw'],\n", + " ['New York City', 'base.popstra.location.vacationers', 'm.0649tj0'],\n", + " ['Kaito Nakamura',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'New York City'],\n", + " ['New York City', 'base.popstra.location.vacationers', 'm.064sr1t'],\n", + " ['A Christmas Caroline', 'book.written_work.subjects', 'New York City'],\n", + " ['Five Towns Jewish Times',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['The Battle for New York: The City at the Heart of the American Revolution',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1fx917'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Aquarian Weekly'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " '13 Going on 30'],\n", + " ['New York City',\n", + " 'base.disneyana.disney_product_theme.disney_products_based_on_this_theme',\n", + " 'New York City Pooh'],\n", + " ['m.063sbhx',\n", + " 'base.petbreeds.dog_city_relationship.cities',\n", + " 'New York City'],\n", + " ['The Approaching 100th Anniversary of the Harlem Renaissance (part 2)',\n", + " 'common.topic.subjects',\n", + " 'New York City'],\n", + " ['New York City Police Department School Safety Division',\n", + " 'organization.organization.place_founded',\n", + " 'New York City'],\n", + " ['Viceroy New York',\n", + " 'location.location.nearby_airports',\n", + " 'John F. Kennedy International Airport'],\n", + " ['Basketball Player', 'common.topic.article', 'm.021_j89'],\n", + " ['Joakim Noah warming up before a Chicago Bulls game',\n", + " 'common.image.size',\n", + " 'm.02bd_n8'],\n", + " ['Brooklyn Bridge', 'transportation.bridge.locale', 'New York City'],\n", + " [\"2006 NCAA Men's Division I Basketball Tournament\",\n", + " 'base.marchmadness.ncaa_basketball_tournament.gender',\n", + " 'Male'],\n", + " ['African American', 'common.topic.webpage', 'm.0b48pfv'],\n", + " ['Chinatown', 'location.neighborhood.neighborhood_of', 'New York City'],\n", + " ['Chrysler Building', 'location.location.containedby', 'Manhattan'],\n", + " ['Catholic Worker', 'book.periodical.language', 'English Language'],\n", + " ['African American', 'common.topic.image', 'AfricanAmericans2'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.includes_causes_of_death',\n", + " 'Myocardial Ischemia'],\n", + " ['m.063lrvg', 'base.popstra.rehab_stay.facility', 'New York City'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'New York International Fringe Festival'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11bccjclv2'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Hair loss'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'New York Post'],\n", + " ['m.064smxq', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['Poly Prep Country Day School',\n", + " 'education.educational_institution.students_graduates',\n", + " 'm.0b6vrkl'],\n", + " ['New York City', 'tv.tv_location.tv_shows_filmed_here', 'Trial Heat'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.0gxvv0q'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Black History Month Enhanced by International Year for People of African Descent'],\n", + " ['Black History Month',\n", + " 'book.book_subject.works',\n", + " 'Black History Month Enhanced by International Year for People of African Descent'],\n", + " ['Joakim Noah', 'people.person.parents', 'Yannick Noah'],\n", + " ['African American',\n", + " 'common.topic.image',\n", + " '1911 Golden Potlatch - Afro-Americans'],\n", + " ['Gastropolis: Food and New York City',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['m.0j13vpp',\n", + " 'base.schemastaging.coach_athlete_relationship.athlete',\n", + " 'Yannick Noah'],\n", + " ['m.0j13l_0', 'people.marriage.spouse', 'Yannick Noah'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1jm96n'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.04j7j5d'],\n", + " ['m.0kfyrv7',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['New York City',\n", + " 'common.topic.subject_of',\n", + " 'The Approaching 100th Anniversary of the Harlem Renaissance (part 2)'],\n", + " ['m.0qh0bg1', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['African American',\n", + " 'base.skosbase.vocabulary_equivalent_topic.narrower_concept',\n", + " 'African American rock musicians'],\n", + " ['m.0107blx6',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['East 34th Street Heliport', 'location.location.containedby', 'New York'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Metropolitan Museum of Art'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Syphilis'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.0z9wk8p'],\n", + " ['Staten Island Advance',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Black History Month',\n", + " 'book.book_subject.works',\n", + " 'Notebook on Black History Month 2012 (Part 6) The Consecrated Soul of Whitney Houston'],\n", + " ['United States of America',\n", + " 'base.locations.countries.continent',\n", + " 'North America'],\n", + " ['Joakim Noah3', 'common.image.appears_in_topic_gallery', 'Joakim Noah'],\n", + " ['Downtown Manhattan Heliport', 'aviation.airport.serves', 'New York City'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Broadway Theatre'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0fkh8y3'],\n", + " [\"Parkinson's disease\", 'medicine.disease.risk_factors', 'Male'],\n", + " ['Adventures by Disney',\n", + " 'travel.tour_operator.travel_destinations',\n", + " 'New York City'],\n", + " ['Tempus Unbound',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'New York City'],\n", + " ['travefy.com',\n", + " 'travel.guidebook.travel_destinations_covered',\n", + " 'New York City'],\n", + " ['United States of America',\n", + " 'base.inspiration.inspiration.inspiration_for',\n", + " '\\\\\"History dressed up in the glow of love’s kiss turned grief into beauty.\\\\\"'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Hirschsprung's disease\"],\n", + " ['New York City Council',\n", + " 'government.governmental_body.members',\n", + " 'm.0x2x5q1'],\n", + " ['Yannick Noah', 'music.group_member.membership', 'm.0w06qbc'],\n", + " ['The Stand', 'tv.tv_program.filming_locations', 'New York City'],\n", + " ['Chelsea Art Museum',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['African American',\n", + " 'people.ethnicity.languages_spoken',\n", + " 'African American Vernacular English'],\n", + " ['New York City Council',\n", + " 'government.governmental_body.members',\n", + " 'm.0w9vg9x'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Little Italy'],\n", + " ['Stroke', 'medicine.disease.risk_factors', 'Transient ischemic attack'],\n", + " ['Chrysler Building',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['Sweden', 'location.country.administrative_divisions', 'Uppsala County'],\n", + " ['Athlete', 'freebase.type_profile.equivalent_topic', 'Athlete'],\n", + " ['Gotham: A History of New York City to 1898',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " [\"2006 NCAA Men's Basketball Championship\",\n", + " 'film.film.genre',\n", + " 'Sports films'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Multiple myeloma'],\n", + " ['Harlem Renaissance',\n", + " 'base.inspiration.inspiration.inspired_by',\n", + " 'African American'],\n", + " ['John F. Kennedy International Airport',\n", + " 'aviation.airport.serves',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'City Hall'],\n", + " ['New York City', 'common.topic.subject_of', 'Viceroy New York'],\n", + " ['Yannick Noah',\n", + " 'music.lyricist.lyrics_written',\n", + " 'What a Different Kind of Mummy'],\n", + " ['Yannick Noah', 'music.artist.album', 'Hommage'],\n", + " ['New York City', 'book.book_subject.works', 'Between Two Rivers: A Novel'],\n", + " ['Thirty-Three Swoons', 'book.written_work.subjects', 'New York City'],\n", + " ['African American',\n", + " 'projects.project_focus.projects',\n", + " 'African American National Biography Project'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Andrew Carnegie Mansion'],\n", + " ['Joakim Noah', 'sports.pro_athlete.teams', 'm.0n819kn'],\n", + " ['New York Golden Blades', 'sports.sports_team.location', 'New York City'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'New York'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Myocardial Ischemia'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_3kf'],\n", + " ['Daily Worker', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Sanitation'],\n", + " ['New York City Department of Design and Construction',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '“September 11, 2001: Citizens of the U.S., besieged by terror’s sting, rose up, weeping glory, as if on eagles’ wings.--from the poem Angel of Remembrance: Candles for September 11, 2001'],\n", + " ['Times Square',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['m.0519ccq',\n", + " 'travel.transportation.transport_terminus',\n", + " 'LaGuardia Airport'],\n", + " ['Notebook on Black History Month 2012 (part 4) The Black Power Mixtape 1967-1975',\n", + " 'book.written_work.subjects',\n", + " 'Black History Month'],\n", + " ['New York City Department of Transportation',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'New York, Empire City: 1920-1945'],\n", + " ['m.09kggfb', 'base.popstra.vacation_choice.location', 'New York City'],\n", + " ['Ellis Island', 'base.usnationalparks.us_national_park.state', 'New York'],\n", + " ['End of 14th Street Crosstown Line',\n", + " 'visual_art.artwork.art_subject',\n", + " 'New York City'],\n", + " ['Chrysler Building', 'location.location.containedby', 'New York City'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0nkjwjp'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Blekinge County'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Tennis Player', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5f5'],\n", + " ['Södermanland County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['Basketball Player', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.010c7kcl', 'sports.sports_award.team', 'Chicago Bulls'],\n", + " ['m.0q845lk', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0zm9_69'],\n", + " ['New York City Half Marathon',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['East Village Other', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pancreatic cancer'],\n", + " ['Viceroy New York',\n", + " 'location.location.nearby_airports',\n", + " 'Newark Liberty International Airport'],\n", + " ['The New Black',\n", + " 'book.book_subject.works',\n", + " 'The Approaching 100th Anniversary of the Harlem Renaissance (part 1)'],\n", + " ['New York City', 'book.book_subject.works', 'The Bonfire of the Vanities'],\n", + " ['The Great Gatsby',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['Uppsala County', 'location.administrative_division.country', 'Sweden'],\n", + " ['New York City',\n", + " 'base.militaryinfiction.location_in_fiction.contained_by',\n", + " 'New York'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Stevie Nichols'],\n", + " ['Yannick Noah', 'music.artist.album', 'Live'],\n", + " ['Central Park', 'location.location.contains', 'Central Park Zoo'],\n", + " ['Staten Island',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area code 917'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvjy3'],\n", + " ['Spanish Language',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'United States of America'],\n", + " ['Delancey School',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Der Groyser Kundes'],\n", + " ['African American',\n", + " 'common.topic.subject_of',\n", + " 'Memory-Song Painted Gold: for the Blue Yusef Lateef (1920-2013)'],\n", + " ['16 Handles Astoria',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['Catholic Worker', 'book.newspaper.circulation_areas', 'New York'],\n", + " ['Teenage Mutant Ninja Turtles',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'China Daily'],\n", + " ['Lemuria',\n", + " 'fictional_universe.fictional_setting.universe',\n", + " 'The Sacred Band of Stepsons universe'],\n", + " ['The Wall Street Journal',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0bm6pqb'],\n", + " ['Darien Lake', 'location.location.containedby', 'New York'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.universe',\n", + " 'Tempus Unbound'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_2gx'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Rockefeller Center'],\n", + " ['Theodore Roosevelt Birthplace National Historic Site',\n", + " 'location.location.containedby',\n", + " 'New York'],\n", + " ['m.051qt6h',\n", + " 'travel.travel_destination_monthly_climate.travel_destination',\n", + " 'New York City'],\n", + " ['m.0n819kn',\n", + " 'sports.sports_team_roster.team',\n", + " \"Florida Gators men's basketball\"],\n", + " ['New York Knicks', 'sports.sports_team.location', 'New York City'],\n", + " ['m.0rfkvx3', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Sweden', 'base.locations.countries.continent', 'Europe'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0rt5nxy'],\n", + " ['New York City',\n", + " 'religion.religious_leadership_jurisdiction.leader',\n", + " 'm.010664km'],\n", + " ['m.0g8qfbh', 'base.firsts.first_achievement.category', 'African American'],\n", + " ['Josh Safdie', 'people.person.place_of_birth', 'New York City'],\n", + " ['Battery Park', 'location.location.containedby', 'New York City'],\n", + " ['Lenny Cooke', 'film.film.film_festivals', '2013 Tribeca Film Festival'],\n", + " ['Sweden', 'location.country.first_level_divisions', 'Östergötland County'],\n", + " ['Sweden',\n", + " 'location.country.administrative_divisions',\n", + " 'Östergötland County'],\n", + " ['The Great Gatsby',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'New York City'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64rh'],\n", + " ['m.0qh0cx8', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'National Academy Museum and School'],\n", + " ['New York City',\n", + " 'base.aareas.schema.administrative_area.subdividing_type',\n", + " 'New York City borough'],\n", + " ['The New York Blade', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Yannick Noah', 'music.lyricist.lyrics_written', 'Yelena'],\n", + " ['Benny Safdie', 'film.editor.film', 'Lenny Cooke'],\n", + " ['Lemurian windows into any place or time',\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Cime'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Västmanland County'],\n", + " ['m.051qt3p',\n", + " 'travel.travel_destination_monthly_climate.travel_destination',\n", + " 'New York City'],\n", + " ['Basketball Player', 'freebase.type_hints.included_types', 'Person'],\n", + " ['Commercial Advertiser',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['New York City', 'travel.travel_destination.climate', 'm.051qt4t'],\n", + " ['Empire State Building', 'location.location.containedby', 'New York City'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Traffic Violations Bureau'],\n", + " ['African American', 'common.topic.webpage', 'm.0b478cr'],\n", + " ['Basketball Player', 'freebase.type_profile.published', 'Published'],\n", + " ['The New York Observer', 'book.periodical.language', 'English Language'],\n", + " ['Times Square Roulette: Remaking the City Icon',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'United Nations Headquarters'],\n", + " ['m.052m20r',\n", + " 'government.government_position_held.governmental_body',\n", + " 'New York City Department of Housing Preservation and Development'],\n", + " ['Esophageal cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Times Square Roulette: Remaking the City Icon',\n", + " 'book.written_work.subjects',\n", + " 'Times Square'],\n", + " ['Nowy Dziennik', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Esophageal cancer'],\n", + " ['New York', 'location.location.containedby', 'United States of America'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'The New Black'],\n", + " ['Benny Safdie', 'people.person.gender', 'Male'],\n", + " ['Staten Island',\n", + " 'base.wikipedia_infobox.settlement.area_code',\n", + " 'Area codes 718, 347 and 929'],\n", + " ['African American', 'common.topic.webpage', 'm.09x81jk'],\n", + " ['New York City Department of Buildings',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['The Sun and the Moon: The Remarkable True Account of Hoaxers, Showmen, Dueling Journalists, and Lunar Man-Bats in Nineteenth-Century New York',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['New York City', 'book.book_subject.works', 'Lowboy'],\n", + " ['Yannick Noah',\n", + " 'music.featured_artist.recordings',\n", + " 'Métis(se) (feat. Yannick Noah)'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Spence School'],\n", + " ['New York Skyports Inc. Seaplane Base',\n", + " 'aviation.airport.serves',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Youth and Community Development'],\n", + " ['The New York Sun', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['m.0qjn30r', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Battery Park',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['New York',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Fire Department'],\n", + " ['Times Square', 'location.location.containedby', 'New York'],\n", + " ['New York City',\n", + " 'visual_art.art_subject.artwork_on_the_subject',\n", + " 'Old-timer, keeping up with the boys'],\n", + " ['New York City', 'book.book_subject.works', 'The Good Life'],\n", + " ['Harlem Renaissance',\n", + " 'book.school_or_movement.associated_works',\n", + " 'The New Black'],\n", + " ['New York Post', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['New York City', 'travel.travel_destination.how_to_get_here', 'm.0519cd5'],\n", + " ['Yannick Noah', 'people.person.gender', 'Male'],\n", + " ['Chrysler Building',\n", + " 'base.newsevents.photograph.photographed_event',\n", + " 'Chrysler Building'],\n", + " ['The Cloisters',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Five Towns Jewish Times', 'book.newspaper.circulation_areas', 'Manhattan'],\n", + " ['\\\\\"The Emancipation Proclamation...can remind us in 2013 of all the mistakes we never want to commit again but it can also motivate us to fulfill to an ever greater degree the definitive freedom-sustaining and life-enhancing principles of democracy in living action.\\\\\"',\n", + " 'media_common.quotation.source',\n", + " 'Notes on the 150th Anniversary of the Emancipation Proclamation'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " \"The Future Once Happened Here: New York, D.C., L.A., and the Fate of America's Big Cities\"],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Passing Through Eden: Photographs of Central Park'],\n", + " ['New York City Police Department School Safety Division',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['New York City', 'travel.travel_destination.how_to_get_here', 'm.0519ccc'],\n", + " ['m.09w163n', 'common.webpage.topic', 'African American'],\n", + " ['Lemurian windows into any place or time',\n", + " 'base.militaryinfiction.location_in_fiction.languages',\n", + " 'English Language'],\n", + " ['\\\\\"History dressed up in the glow of love’s kiss turned grief into beauty.\\\\\"',\n", + " 'media_common.quotation.subjects',\n", + " 'United States of America'],\n", + " ['The Bonfire of the Vanities',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnx7'],\n", + " ['Basketball player',\n", + " 'base.skosbase.vocabulary_equivalent_topic.equivalent_concept',\n", + " 'Basketball players'],\n", + " ['Sedan', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Sheldon Hawkes',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.governing_officials',\n", + " 'm.029837n'],\n", + " ['The Stand', 'tv.tv_program.languages', 'English Language'],\n", + " ['Basketball player',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010h53zd'],\n", + " ['Statue of Liberty National Monument',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Black History Month Enhanced by International Year for People of African Descent',\n", + " 'book.written_work.subjects',\n", + " 'African diaspora'],\n", + " ['m.0wg9gt4',\n", + " 'location.partial_containment_relationship.partially_contains',\n", + " 'New York City'],\n", + " ['Halland County', 'location.administrative_division.country', 'Sweden'],\n", + " ['New York City', 'business.employer.employees', 'm.0123w_43'],\n", + " ['SEO Services New York',\n", + " 'location.location.time_zones',\n", + " 'Eastern Time Zone'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Gotham: A History of New York City to 1898'],\n", + " ['International Center of Photography',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Richie', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Buildings'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Amyotrophic lateral sclerosis'],\n", + " ['\\\\\"History dressed up in the glow of love’s kiss turned grief into beauty.\\\\\"',\n", + " 'base.inspiration.inspiration.inspired_by',\n", + " 'United States of America'],\n", + " ['Norrbotten County', 'location.location.containedby', 'Sweden'],\n", + " ['Vanishing Point: The Disappearance of Judge Crater, and the New York He Left Behind',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['Joakim Noah', 'basketball.basketball_player.position_s', 'Center'],\n", + " ['m.09wwqj3', 'common.webpage.topic', 'African American'],\n", + " ['The National Sports Daily',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['High Line',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['One Brick', 'organization.organization.geographic_scope', 'New York City'],\n", + " ['Area code 917', 'location.location.containedby', 'New York'],\n", + " ['Manhattan',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Spence School'],\n", + " ['m.0g8qg56', 'base.firsts.first_achievement.category', 'African American'],\n", + " ['Male',\n", + " 'base.skosbase.vocabulary_equivalent_topic.equivalent_concept',\n", + " 'Males'],\n", + " ['New York City Law Department',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['m.05bvk3z',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Environmental Protection'],\n", + " ['New York Skyports Inc. Seaplane Base',\n", + " 'aviation.airport.serves',\n", + " 'New York'],\n", + " ['Songs of My People',\n", + " 'exhibitions.exhibition.subjects',\n", + " 'Harlem Renaissance'],\n", + " ['m.0drxd0b', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Crocheron Park', 'location.location.containedby', 'New York City'],\n", + " ['New York City', 'book.book_subject.works', 'The Moneychangers'],\n", + " ['Traffic Violations Bureau',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Human Resources Administration'],\n", + " ['Al-Hoda', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['African American',\n", + " 'book.periodical_subject.periodicals',\n", + " 'African American Review'],\n", + " ['New Yorker Volkszeitung',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Schomburg Center for Research in Black Culture',\n", + " 'location.location.containedby',\n", + " 'New York'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'New York City Half Marathon'],\n", + " ['m.02397ys', 'sports.sports_team_roster.team', 'Chicago Bulls'],\n", + " ['Ellis Island', 'location.location.partially_containedby', 'New York City'],\n", + " ['The London 2012 Summer Olympics',\n", + " 'olympics.olympic_games.sports',\n", + " 'Basketball'],\n", + " ['m.0qb8t60', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Harlem Renaissance Way Down South',\n", + " 'book.written_work.original_language',\n", + " 'American English'],\n", + " ['m.010_xz6_', 'freebase.valuenotation.has_no_value', 'Nominated work'],\n", + " ['Cécilia Rodhe', 'people.person.nationality', 'Sweden'],\n", + " ['m.04j7jc7',\n", + " 'government.government_position_held.governmental_body',\n", + " 'New York City Housing Authority'],\n", + " ['Area codes 212 and 646',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wg9gsz'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Little Italy'],\n", + " ['m.0f88bjv', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0rsqxq3'],\n", + " ['Joakim Noah', 'people.person.nationality', 'France'],\n", + " ['Cime',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Tempus Unbound'],\n", + " ['m.0wffpj8',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'New York City'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5gr'],\n", + " ['Joakim Noah', 'sports.sports_award_winner.awards', 'm.010c7kcl'],\n", + " ['New York City',\n", + " 'government.governmental_jurisdiction.agencies',\n", + " 'New York City Department of Homeless Services'],\n", + " ['Center', 'common.topic.notable_for', 'g.125br8h6w'],\n", + " ['The Moneychangers', 'book.written_work.subjects', 'New York City'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Times Square Spectacular: Lighting up Broadway'],\n", + " ['Joakim Noah',\n", + " 'common.topic.image',\n", + " 'Joakim Noah warming up before a Chicago Bulls game'],\n", + " ['m.070ztkj', 'base.popstra.arrest.location', 'New York City'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Notes on the 150th Anniversary of the Emancipation Proclamation'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Battery Park'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1m372j'],\n", + " ['Dalarna County', 'location.location.containedby', 'Sweden'],\n", + " ['Female', 'medicine.risk_factor.diseases', 'Kyphosis'],\n", + " ['m.04hskl8', 'people.place_lived.person', 'Yannick Noah'],\n", + " ['Museum of Arts and Design',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Times Square',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['SFX Award for Best Actor',\n", + " 'award.award_category.disciplines_or_subjects',\n", + " 'Male'],\n", + " [\"Florida Gators men's basketball\",\n", + " 'sports.sports_team.roster',\n", + " 'm.0n819kn'],\n", + " ['The New York Times', 'book.newspaper.circulation_areas', 'Manhattan'],\n", + " ['Earth', 'base.locations.planets.continents_within', 'Europe'],\n", + " ['New York',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['New York Knicks', 'sports.sports_team.sport', 'Basketball'],\n", + " ['m.063sbjd',\n", + " 'base.petbreeds.dog_city_relationship.cities',\n", + " 'New York City'],\n", + " ['New York City', 'travel.travel_destination.climate', 'm.051qt6h'],\n", + " ['New York City',\n", + " 'location.location.nearby_airports',\n", + " 'John F. Kennedy International Airport'],\n", + " ['Black History Month',\n", + " 'book.book_subject.works',\n", + " 'Notebook on Black History Month 2012 (Part 1): Carter G. Woodson and Company'],\n", + " ['New York City', 'travel.travel_destination.climate', 'm.051qt66'],\n", + " ['City Hall', 'media_common.netflix_title.netflix_genres', 'Drama'],\n", + " ['The Sacred Band of Stepsons universe',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " 'Lemurian windows into any place or time'],\n", + " ['The Approaching 100th Anniversary of the Harlem Renaissance (part 1)',\n", + " 'book.written_work.school_or_movement',\n", + " 'African American'],\n", + " ['Black History Month',\n", + " 'base.celebrations.celebration_subject.event',\n", + " 'Black History Month'],\n", + " ['Lemuria',\n", + " 'fictional_universe.type_of_fictional_setting.settings',\n", + " 'New York City'],\n", + " ['Earth',\n", + " 'base.locations.planets.countries_within',\n", + " 'United States of America'],\n", + " ['African American', 'common.topic.webpage', 'm.09x2nkc'],\n", + " ['New York',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Daily Worker'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04mmfr8'],\n", + " ['Staten Island',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Staten Island Advance'],\n", + " ['Spence School',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['International Center of Photography',\n", + " 'location.location.containedby',\n", + " 'New York City'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.0kfyrt3'],\n", + " ['m.010n_57x', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Yannick Noah', 'music.group_member.membership', 'm.0wd68jy'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5h5'],\n", + " ['The Wisdom of W.E.B. Du Bois',\n", + " 'book.written_work.subjects',\n", + " 'Harlem Renaissance'],\n", + " ['Staten Island',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The New York Times'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Cardiovascular disease'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'New York'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0dzbw6f'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pyloric stenosis'],\n", + " ['m.0nkjx3t', 'music.track_contribution.contributor', 'Yannick Noah'],\n", + " ['Yannick Noah', 'music.artist.album', 'Yannick Noah'],\n", + " ['Generous Enemies: Patriots and Loyalists in Revolutionary New York',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['m.010n_4mv', 'film.film_regional_release_date.film', 'Lenny Cooke'],\n", + " ['Best NBA Player ESPY Award',\n", + " 'award.award_category.nominees',\n", + " 'm.010_xz6_'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Ellis Island'],\n", + " ['New York City', 'location.statistical_region.population', 'g.1jmcbgvzj'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1l4_tt'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvkm6'],\n", + " ['Manhattan',\n", + " 'location.place_with_neighborhoods.neighborhoods',\n", + " 'Chinatown'],\n", + " ['Sweden',\n", + " 'location.country.administrative_divisions',\n", + " 'Västmanland County'],\n", + " ['New York City',\n", + " 'book.book_subject.musical_compositions_about_this_topic',\n", + " 'Time'],\n", + " ['African American', 'common.topic.webpage', 'm.0b46zq1'],\n", + " ['Joakim Noah 2', 'common.image.size', 'm.05tyn3c'],\n", + " ['Joakim Noah', 'people.person.sibling_s', 'm.0j2hqb0'],\n", + " ['Ellis Island',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['Västernorrland County', 'location.location.containedby', 'Sweden'],\n", + " ['African American', 'common.topic.webpage', 'm.09wsrjm'],\n", + " ['New York Knicks',\n", + " 'sports.sports_team.arena_stadium',\n", + " 'Madison Square Garden'],\n", + " ['New York City Housing Authority',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['m.0blp5sr', 'biology.hybrid_parentage.parent_sex', 'Male'],\n", + " ['Östergötland County', 'location.location.containedby', 'Sweden'],\n", + " ['LaGuardia Airport',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Lenny Cooke', 'film.film.executive_produced_by', 'Adam Shopkorn'],\n", + " ['Area codes 212 and 646', 'location.location.containedby', 'New York City'],\n", + " ['Basketball Player', 'type.type.expected_by', 'Basketball player'],\n", + " ['Spirit of the Times', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Writing New York: A Literary Anthology',\n", + " 'book.written_work.subjects',\n", + " 'New York City'],\n", + " ['Kronoberg County', 'location.administrative_division.country', 'Sweden'],\n", + " ['New York City Housing Authority',\n", + " 'government.governmental_body.jurisdiction',\n", + " 'New York City'],\n", + " ['Yannick Noah', 'people.person.parents', 'Zacharie Noah'],\n", + " ['Staten Island',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States of America'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0dqm8wr'],\n", + " ['Museum of Modern Art',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Basketball player',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Cam Calloway'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0fjw72l'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.events',\n", + " 'Askelon follows Cime to the 20th Century'],\n", + " ['m.0g8qg34', 'base.firsts.first_achievement.category', 'African American'],\n", + " ['Yannick Noah', 'music.group_member.membership', 'm.0vyrky6'],\n", + " ['Peter Clemenza', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Västmanland County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " [\"The Russian Debutante's Handbook\",\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['The Approaching 100th Anniversary of the Harlem Renaissance (part 1)',\n", + " 'book.written_work.subjects',\n", + " 'The New Black'],\n", + " ['Jönköping County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['LaGuardia Airport', 'aviation.airport.serves', 'Viceroy New York'],\n", + " ['New York City',\n", + " 'base.webvideo.internet_video_genre.series',\n", + " 'Second-Hand NY'],\n", + " ['m.0wg91t7',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'New York City'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Benign prostatic hyperplasia'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'American Museum of Natural History'],\n", + " ['Athlete', 'freebase.type_profile.published', 'Published'],\n", + " ['New York City', 'book.book_subject.works', 'Art Deco New York'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Dana Barrett'],\n", + " ['Yannick Noah', 'people.person.spouse_s', 'm.0j13l_c'],\n", + " ['m.03p9kc1',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'New York City'],\n", + " ['The Cloisters', 'location.location.containedby', 'New York'],\n", + " ['New York City',\n", + " 'location.hud_foreclosure_area.estimated_number_foreclosures',\n", + " 'm.07f62rg'],\n", + " ['Lenny Cooke',\n", + " 'film.film.film_festivals',\n", + " '2013 Venice International Film Festival'],\n", + " ['Athlete', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['United States Dollar',\n", + " 'base.coinsdaily.coin_type.country',\n", + " 'United States of America'],\n", + " ['Notes on the 150th Anniversary of the Emancipation Proclamation',\n", + " 'book.written_work.school_or_movement',\n", + " 'African American'],\n", + " ['Basketball Player', 'type.type.properties', 'Career Averages'],\n", + " ['New York Evening Mail',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Chicago Bulls', 'sports.sports_team.roster', 'm.02397ys'],\n", + " ['m.0107blsl', 'film.personal_film_appearance.film', 'Lenny Cooke'],\n", + " ['MBSF - Private Jets',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Västerbotten County'],\n", + " ['Ellis Island',\n", + " 'location.location.containedby',\n", + " 'Statue of Liberty National Monument'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0f8zmwb'],\n", + " ['Yannick Noah', 'people.person.profession', 'Tennis player'],\n", + " ['m.046cvpy', 'location.adjoining_relationship.adjoins', 'New York City'],\n", + " ['The Sun', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Sesame Street',\n", + " 'tv.tv_program.country_of_origin',\n", + " 'United States of America'],\n", + " ['France', 'location.location.containedby', 'Europe'],\n", + " ['Yannick Noah', 'music.artist.album', 'Métis(se)'],\n", + " ['Kronoberg County', 'location.location.containedby', 'Sweden'],\n", + " ['Richie',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'New York City'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1jdv9v'],\n", + " ['African American',\n", + " 'book.school_or_movement.associated_works',\n", + " \"Report on 2011 International Year part 4 Haiti's Hope Now and Tomorrow\"],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0rjhq58'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Sullivan & Galleshaw, LLP'],\n", + " ['Peak Sales Recruiting',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['The Jewish Daily Forward',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['New York',\n", + " 'business.business_location.parent_company',\n", + " 'MBSF - Private Jets'],\n", + " ['Lush Life: A Novel', 'book.written_work.subjects', 'Manhattan'],\n", + " ['Mitchel Air Force Base', 'aviation.airport.serves', 'New York City'],\n", + " ['Freedomland U.S.A.',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['Lemurian windows into any place or time',\n", + " 'fictional_universe.fictional_setting.contained_by',\n", + " 'Lemuria'],\n", + " ['Basketball Player', 'type.type.properties', 'Player Statistics'],\n", + " ['Shreve, Lamb & Harmon',\n", + " 'architecture.architecture_firm.projects',\n", + " 'Empire State Building'],\n", + " ['New York Yankees', 'sports.sports_team.location', 'New York City'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Stroke'],\n", + " ['New York City', 'location.location.partiallycontains', 'm.0wg91t7'],\n", + " ['Södermanland County',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'Sweden'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Brooklyn Bridge'],\n", + " ['New York City',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'George Gustav Heye Center'],\n", + " ['New York City', 'location.statistical_region.population', 'm.0hlr5cn'],\n", + " ['Actor', 'common.topic.notable_types', 'Profession'],\n", + " ['Manhattan',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'New York City'],\n", + " ['United States of America',\n", + " 'location.location.time_zones',\n", + " 'Eastern Time Zone'],\n", + " ['The Wall Street Journal',\n", + " 'book.newspaper.circulation_areas',\n", + " 'Staten Island'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Death at the Old Hotel: A Bartender Brian McNulty Mystery'],\n", + " ['Gothenburg', 'location.location.containedby', 'Västra Götaland County'],\n", + " ['Joakim Noah', 'olympics.olympic_athlete.country', 'm.0k8lwgq'],\n", + " ['m.010h53v9',\n", + " 'government.government_position_held.jurisdiction_of_office',\n", + " 'New York City'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Daily News'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Uppsala County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['m.09xsbrc', 'common.webpage.topic', 'African American'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Leprosy'],\n", + " ['New York City',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Mac Taylor'],\n", + " ['New York City Department of Consumer Affairs',\n", + " 'government.government_agency.jurisdiction',\n", + " 'New York City'],\n", + " ['m.09ys4my', 'common.webpage.topic', 'African American'],\n", + " ['New York City', 'location.statistical_region.population', 'm.064gnql'],\n", + " ['Deluge', 'film.film.language', 'English Language'],\n", + " ['Yannick Noah', 'music.artist.track_contributions', 'm.0qd8ppn'],\n", + " ['Center',\n", + " 'common.topic.image',\n", + " 'Stefania Passaro taglia fuori Uljana Semjonova'],\n", + " ['Yélena Noah', 'people.person.sibling_s', 'm.0j2hqb0'],\n", + " ['Lowboy', 'book.written_work.subjects', 'New York City Subway'],\n", + " ['Jo',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'New York City'],\n", + " ['Tempus Unbound',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Kansas'],\n", + " ['New York City', 'location.statistical_region.population', 'g.11x1j6cs5'],\n", + " ['New York Perl Mongers',\n", + " 'organization.organization.geographic_scope',\n", + " 'New York City'],\n", + " ['Notes on the 150th Anniversary of the Emancipation Proclamation',\n", + " 'media_common.quotation_source.quotations',\n", + " '\\\\\"The Emancipation Proclamation...can remind us in 2013 of all the mistakes we never want to commit again but it can also motivate us to fulfill to an ever greater degree the definitive freedom-sustaining and life-enhancing principles of democracy in living action.\\\\\"'],\n", + " ['Staten Island', 'location.us_county.hud_county_place', 'New York City'],\n", + " ['Yannick Noah',\n", + " 'tennis.tennis_tournament_champion.tennis_titles',\n", + " 'm.09vqf06'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'New York Evening Mail'],\n", + " ['Uppsala County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['Amyotrophic lateral sclerosis', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Sedan', 'location.location.people_born_here', 'Yannick Noah'],\n", + " ['Joakim Noah', 'award.award_nominee.award_nominations', 'm.010_xz6_'],\n", + " ['Skåne County',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Sweden'],\n", + " ['Lenny Cooke', 'film.film.release_date_s', 'm.010n_9hh'],\n", + " ['Yannick Noah', 'common.topic.article', 'm.05qr5d'],\n", + " ['American Museum of Natural History',\n", + " 'location.location.containedby',\n", + " 'New York City'],\n", + " ['MBSF - Private Jets', 'location.location.containedby', 'New York City'],\n", + " ['Freedomland U.S.A.', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['New York City',\n", + " 'organization.organization_scope.organizations_with_this_scope',\n", + " 'Alchemical Theatre'],\n", + " ['Staten Island',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['Tesla Science Center at Wardenclyffe',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York City'],\n", + " ['m.010p01ry', 'film.film_crew_gig.film', 'Lenny Cooke'],\n", + " ['High Line',\n", + " 'travel.tourist_attraction.near_travel_destination',\n", + " 'New York'],\n", + " ['Freie Arbeiter Stimme',\n", + " 'book.newspaper.circulation_areas',\n", + " 'New York City'],\n", + " ['Lemuria',\n", + " 'base.militaryinfiction.location_in_fiction.languages',\n", + " 'English Language'],\n", + " ['Male', 'common.topic.image', 'male.jpg'],\n", + " ['Viceroy New York', 'common.topic.subject_of', 'Manhattan'],\n", + " ['Grand Central Terminal', 'location.location.containedby', 'New York City'],\n", + " ['Joakim Noah', 'people.person.education', 'm.040rzsq'],\n", + " ['Jönköping County', 'location.administrative_division.country', 'Sweden'],\n", + " ['New York-New York Hotel and Casino',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['m.010q9zv3', 'award.award_honor.award', 'NBA All-Defensive Team'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Abdominal aortic aneurysm'],\n", + " ['Manhattan',\n", + " 'common.topic.image',\n", + " 'Midtown Manhattan as seen from the GE Building.'],\n", + " ['New York City', 'location.location.partially_contains', 'Ellis Island'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Morgen Freiheit'],\n", + " ['m.0wffpyb', 'film.film_crew_gig.film', 'Lenny Cooke'],\n", + " ['Central Park',\n", + " 'book.book_subject.works',\n", + " 'Central Park in the Dark: More Mysteries of Urban Wildlife'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'The Cloisters'],\n", + " ['Sweden', 'location.country.first_level_divisions', 'Värmland County'],\n", + " ['Joakim Noah', 'common.topic.notable_for', 'g.1259vy5mg'],\n", + " ['Cardiovascular disease',\n", + " 'medicine.disease_cause.diseases',\n", + " 'Transient ischemic attack'],\n", + " ['New York City borough',\n", + " 'base.aareas.schema.administrative_area_type.subdivides_place',\n", + " 'New York City'],\n", + " ['New York City', 'location.location.partially_contained_by', 'm.0wg9gt4'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Chelsea Clinton News'],\n", + " ['United States of America',\n", + " 'location.country.languages_spoken',\n", + " 'Spanish Language'],\n", + " ['m.010_xz6_', 'award.award_nomination.award', 'Best NBA Player ESPY Award'],\n", + " ['New York City',\n", + " 'location.hud_foreclosure_area.ofheo_price_change',\n", + " 'm.07f62sj'],\n", + " ['m.010_xz6_', 'award.award_nomination.award_nominee', 'Joakim Noah'],\n", + " ['Sweden',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Kalmar County'],\n", + " ['New York City',\n", + " 'law.court_jurisdiction_area.courts',\n", + " 'United States Bankruptcy Court for the Southern District of New York'],\n", + " ['Joakim Noah', 'people.person.ethnicity', 'African American'],\n", + " ['National Guardian', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['Blekinge County',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'Sweden'],\n", + " ['heart attack',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Abdominal aortic aneurysm'],\n", + " ['African American', 'common.topic.webpage', 'm.09y2t_h'],\n", + " ['New York Press', 'book.newspaper.circulation_areas', 'Manhattan'],\n", + " ['Tennis player', 'people.profession.specialization_of', 'Athlete'],\n", + " ['Center', 'sports.sports_position.sport', 'Basketball'],\n", + " ['New York City',\n", + " 'book.book_subject.works',\n", + " 'Songs of My People Art Exhibition Opens at Penn Center'],\n", + " ...],\n", + " [['Jay Gruden', 'sports.pro_athlete.teams', 'm.0hpq60f'],\n", + " ['Washington Redskins Helmet', 'common.image.size', 'm.01x5zr3'],\n", + " ['m.0793ny9',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['St. Louis Rams at Washington Redskins, 2009-09-20',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07mlbn_'],\n", + " ['Jay Gruden', 'people.person.education', 'm.0hpn6b3'],\n", + " ['m.09rdm0n',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09tp7mj',\n", + " 'organization.organization_board_membership.member',\n", + " 'Dwight Schar'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09g3cc9'],\n", + " ['m.07jqjn6',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qhs'],\n", + " ['m.096q4pr',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Jason Campbell', 'sports.drafted_athlete.drafted', 'm.05bv768'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.receiving',\n", + " 'm.08nl7wd'],\n", + " ['NFC East', 'base.schemastaging.context_name.pronunciation', 'g.125_qxvkq'],\n", + " ['m.0793plw',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07vr595',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09rdlzv',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Diego Chargers'],\n", + " ['m.075ft5q',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.04xh13y',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Bill McPeak'],\n", + " ['m.0793qyw',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['George Preston Marshall',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'Washington Redskins'],\n", + " ['m.0v317s8', 'sports.sports_league_draft_pick.draft', '2013 NFL draft'],\n", + " ['m.0793p7t',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2008-10-05'],\n", + " ['m.075ft10',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793q5l'],\n", + " ['Arizona Cardinals',\n", + " 'american_football.football_team.away_games',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['m.04xh12k',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['m.09g3cbz',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['Jay Gruden',\n", + " 'american_football.football_player.forty_yard_dash_time',\n", + " 'm.0x15j0c'],\n", + " ['m.07p5wrj',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.away_team',\n", + " 'Washington Redskins'],\n", + " ['m.07927vl',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Cleveland Browns'],\n", + " ['m.09g3cbm',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.098jl4p',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['Daniel Snyder', 'freebase.valuenotation.has_value', 'Children'],\n", + " ['m.09knjq5',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794njj'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftfw'],\n", + " ['m.075ftkr',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.0794nj0',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qs7'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09knjrx'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jl4p'],\n", + " ['Super Bowl XXVI',\n", + " 'sports.sports_championship_event.championship',\n", + " 'Super Bowl'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.passing',\n", + " 'm.098jl2w'],\n", + " ['m.0793nvl',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['m.0hpq60f', 'sports.sports_team_roster.player', 'Jay Gruden'],\n", + " ['m.09rdlt5',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['m.010dsp_6', 'internet.localized_uri.language', 'English'],\n", + " ['m.0793r7m',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Baltimore Ravens, 2008-12-07'],\n", + " ['m.096q484',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['Bill Austin', 'people.person.nationality', 'United States of America'],\n", + " ['George Preston Marshall', 'people.person.gender', 'Male'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.passing',\n", + " 'm.0792c0t'],\n", + " ['Washington Redskins',\n", + " 'sports.sports_team.previously_known_as',\n", + " 'Boston Redskins'],\n", + " ['Jason Campbell', 'people.person.nationality', 'United States of America'],\n", + " ['m.0793r8b',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['m.07t471p',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['m.096q4tb',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['LaRon Landry', 'common.topic.notable_types', 'American football player'],\n", + " ['m.075ftf3', 'sports.sports_team_season_record.season', '1964 NFL season'],\n", + " ['m.0793r9k',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.passing',\n", + " 'm.08nl7x5'],\n", + " ['m.0793plc',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07t473q',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.subsumes',\n", + " 'United States of America'],\n", + " ['Washington Redskins', 'common.topic.article', 'm.084lh'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.passing',\n", + " 'm.0793py3'],\n", + " ['m.075ftgw', 'sports.sports_team_season_record.season', '1957 NFL season'],\n", + " ['m.0793xpk',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2008-10-26'],\n", + " ['m.0794nhp',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Arizona Cardinals'],\n", + " ['Baltimore Ravens', 'common.topic.notable_types', 'American football team'],\n", + " ['m.0793gbj',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['George Preston Marshall',\n", + " 'people.person.parents',\n", + " 'Thomas Hildebrand Marshall'],\n", + " ['Washington Redskins',\n", + " 'sports.sports_team.championships',\n", + " 'Super Bowl XVII'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.home_team',\n", + " 'Washington Redskins'],\n", + " ['m.09rdlz4',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Diego Chargers'],\n", + " ['m.0793pql',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793mjd'],\n", + " ['m.0793q24',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2008-10-26'],\n", + " ['m.075ftgc',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.09g3cc9',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07927xg',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0901txt'],\n", + " ['m.088w_q1',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Atlanta Falcons'],\n", + " ['m.09g3cdb',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.075ftb8', 'sports.sports_team_season_record.season', '1975 NFL season'],\n", + " ['Burgundy', 'common.topic.notable_types', 'Color'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.away_games',\n", + " 'Washington Redskins at Dallas Cowboys, 2008-09-28'],\n", + " ['m.09k0gg3',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['FedExField', 'sports.sports_facility.home_venue_for', 'm.0n5zj4p'],\n", + " ['m.07vr5gq',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Kansas City Chiefs'],\n", + " ['San Diego Chargers',\n", + " 'sports.sports_team.arena_stadium',\n", + " 'Qualcomm Stadium'],\n", + " ['Kansas City Chiefs at Washington Redskins, 2009-10-18',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07vr5fk'],\n", + " ['m.0793ggh',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07jqjbv'],\n", + " ['m.098jl0g',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09knjnq',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793mcp',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794brd'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09g3c6d'],\n", + " ['m.07938zy',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2008-09-28'],\n", + " ['New York Giants at Washington Redskins, 2008-11-30',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793gdx'],\n", + " ['m.07ybyst',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['m.07950pm',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Seattle Seahawks'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07t4778'],\n", + " ['m.0793rdy',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['m.05lkf5n', 'education.education.student', 'Daniel Snyder'],\n", + " ['m.0793r1c',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.receiving',\n", + " 'm.096q4r4'],\n", + " ['m.0794yws',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Francisco 49ers'],\n", + " ['m.04xh13k',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793p3h',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Tiffin', 'location.hud_county_place.place', 'Tiffin'],\n", + " ['Carolina Panthers', 'sports.sports_team.sport', 'American football'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.receiving',\n", + " 'm.088x023'],\n", + " ['Mike Shanahan',\n", + " 'sports.sports_team_coach.sports_coached',\n", + " 'American football'],\n", + " ['m.0793mgc',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07t471p'],\n", + " ['The PopWatch Confessional (Vol. 7)',\n", + " 'common.resource.annotations',\n", + " 'm.09xvg0d'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075fthz'],\n", + " ['American football player',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['m.07927x1',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['m.07nn47v',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['m.0793rk3',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['m.09rdlxs',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Diego Chargers'],\n", + " ['m.07jqj91',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New York Giants'],\n", + " ['NFC West', 'organization.organization.parent', 'm.0ck2dx2'],\n", + " ['Super Bowl', 'freebase.equivalent_topic.equivalent_type', 'Super bowl'],\n", + " ['m.0793r2z',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['Green Bay Packers',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['National Football Conference',\n", + " 'organization.organization.child',\n", + " 'm.0ck2dxg'],\n", + " ['m.07927s0',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Cleveland Browns'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.historical_coaching_staff',\n", + " 'm.04xh139'],\n", + " ['m.0793ryc',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['m.07925wq',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Cincinnati Bengals'],\n", + " ['White',\n", + " 'base.roses.rose_color.roses_of_this_color',\n", + " \"Rosa 'Ivory Fashion'\"],\n", + " ['m.0901vcz',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2009-11-22'],\n", + " ['m.075ft5_', 'sports.sports_team_season_record.season', '1990 NFL season'],\n", + " ['m.0793pft',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.historical_coaching_staff',\n", + " 'm.04xh155'],\n", + " ['Super Bowl',\n", + " 'sports.sports_championship.league',\n", + " 'National Football League'],\n", + " ['m.08nl7rx',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['m.07nn46c',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['m.0794bnb',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New Orleans Saints'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ft20'],\n", + " ['Gold', 'common.topic.notable_for', 'g.1254yr824'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.rushing',\n", + " 'm.098jkzf'],\n", + " ['Washington Redskins at Dallas Cowboys, 2008-09-28',\n", + " 'american_football.football_game.away_team',\n", + " 'Washington Redskins'],\n", + " ['m.075ft9j',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.088w_yf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0g9t'],\n", + " ['m.0901v8m',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['Curly Lambeau', 'common.topic.notable_types', 'American football player'],\n", + " ['Louisiana State University',\n", + " 'location.location.containedby',\n", + " 'United States of America'],\n", + " ['NFL Greatest Super Bowl Moments: I-XLI',\n", + " 'film.film.personal_appearances',\n", + " 'm.0jbcqvk'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.away_games',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['Super Bowl XXVI',\n", + " 'sports.sports_championship_event.champion',\n", + " 'Washington Redskins'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'common.topic.notable_for',\n", + " 'g.125c7g_0p'],\n", + " ['m.0794bpd',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['m.0793ncy',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09g3cb8'],\n", + " ['m.088w_tk',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.0793qlw',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['Owner(s)', 'type.property.schema', 'Professional Sports Team'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.passing',\n", + " 'm.0793ph6'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qf_'],\n", + " ['m.0px_kgz',\n", + " 'base.schemastaging.phone_sandbox.service_location',\n", + " 'United States of America'],\n", + " ['m.04xh155',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Steve Spurrier'],\n", + " ['m.08nl7ny',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.098jkzf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07p5wkf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07nn4gj',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Jason Campbell', 'people.person.gender', 'Male'],\n", + " ['m.07vr5b6',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Kansas City Chiefs at Washington Redskins, 2009-10-18'],\n", + " ['Dwight Schar', 'common.topic.webpage', 'm.0460vpt'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793r2g'],\n", + " ['m.07jqjd6',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['m.05_g5z9', 'business.sponsorship.sponsored_recipient', 'FedExField'],\n", + " ['m.0793rb2',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Sports League Championship Event',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Daniel Snyder', 'people.person.employment_history', 'm.05lkg14'],\n", + " ['m.09g3c50',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jl1h'],\n", + " ['m.075ftfc', 'sports.sports_team_season_record.season', '1963 NFL season'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07nn48w'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.rushing',\n", + " 'm.096q484'],\n", + " ['m.0px_fmz',\n", + " 'base.schemastaging.phone_sandbox.caller_category',\n", + " 'Premium Customer Service'],\n", + " ['m.05bv768', 'sports.sports_league_draft_pick.school', 'Auburn University'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt9cq'],\n", + " ['m.0793qs7',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['Cleveland Browns at Washington Redskins, 2008-10-19',\n", + " 'american_football.football_game.passing',\n", + " 'm.07927rj'],\n", + " ['Daniel Snyder', 'people.person.gender', 'Male'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793r5r'],\n", + " ['m.0793xm5',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['Buffalo Bills', 'common.topic.notable_types', 'Professional Sports Team'],\n", + " ['m.07950n8',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['m.0793k02',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2008-10-05'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793q46'],\n", + " ['m.07ybypr',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0793qz7',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Detroit Lions', 'sports.sports_team.league', 'm.0crt9b6'],\n", + " ['Detroit Lions',\n", + " 'american_football.football_team.home_games',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['m.088w_zx',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.096q4c1',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.07jqjm8',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['Jay Gruden',\n", + " 'sports.pro_athlete.sports_played_professionally',\n", + " 'm.0d5p7wv'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794yzd'],\n", + " ['Jay Gruden', 'people.person.parents', 'Jim Gruden'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qqm'],\n", + " ['m.07mlblv',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['m.0793qkv',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Philadelphia Eagles',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['m.0792c1l',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Pittsburgh Steelers at Washington Redskins, 2008-11-03'],\n", + " ['m.0793qp9',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Tampa Bay Buccaneers at Washington Redskins, 2009-10-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07p5wlv'],\n", + " ['m.0793j_d',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0793pzt',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Marty Schottenheimer',\n", + " 'american_football.football_coach.coaching_history',\n", + " 'm.04xh151'],\n", + " ['Sean Taylor', 'people.person.gender', 'Male'],\n", + " ['m.0795357',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'St. Louis Rams'],\n", + " ['m.07927wb',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Cleveland Browns'],\n", + " ['m.0793nhh',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Washington, D.C.'],\n", + " ['Jim Zorn',\n", + " 'american_football.football_coach.coaching_history',\n", + " 'm.0vsc6bj'],\n", + " ['m.0793rk3',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Cowboys–Redskins rivalry',\n", + " 'base.rivalries.rivalry.rival',\n", + " 'Dallas Cowboys'],\n", + " ['m.075ft39', 'sports.sports_team_season_record.season', '1999 NFL season'],\n", + " ['m.09g3c72',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.home_team',\n", + " 'Washington Redskins'],\n", + " ['m.09knjsl',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.0793mf2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0901tzf',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2009-11-22'],\n", + " ['m.07925wq',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0792c3v'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0901t_7'],\n", + " ['m.07t472q',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['m.0793q_4',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.away_team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07nn4j7'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793d0n'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07t476l'],\n", + " ['m.07ybyn6',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['m.07nn4h6',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Professional Sports Team',\n", + " 'freebase.type_hints.included_types',\n", + " 'Sports Team'],\n", + " ['m.0793r3t',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793qq2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Super Bowl XVII', 'common.topic.notable_types', 'Super bowl'],\n", + " ['m.07ybyqr',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0793pft',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2008-10-05'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.season',\n", + " '2008 NFL season'],\n", + " ['New York Giants at Washington Redskins, 2008-11-30',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793qwr'],\n", + " ['m.09k0gcq',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['San Diego Chargers',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['1982 NFL season',\n", + " 'sports.sports_league_season.championship',\n", + " 'Super Bowl XVII'],\n", + " ['m.0793nwf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793d1f'],\n", + " ['m.09k0g6q',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['m.0792c0t',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Pittsburgh Steelers'],\n", + " ['New York Giants',\n", + " 'american_football.football_team.away_games',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['m.07jqjbv',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['m.075ft7v',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793r6q',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0crt98k',\n", + " 'sports.sports_league_participation.team',\n", + " 'Arizona Cardinals'],\n", + " ['Eddie Casey', 'people.person.gender', 'Male'],\n", + " ['m.09g3c4p',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.04xh12p',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['Minnesota Vikings', 'sports.sports_team.sport', 'American football'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794bnb'],\n", + " ['m.0crt504',\n", + " 'sports.sports_league_participation.league',\n", + " 'National Football League'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07925vz'],\n", + " ['m.088w_zg',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09rdm2r'],\n", + " ['m.0ztjz9h', 'sports.sports_team_coach_tenure.coach', 'Mike Shanahan'],\n", + " ['Philadelphia Eagles',\n", + " 'american_football.football_team.division',\n", + " 'NFC East'],\n", + " ['2013 NFL draft', 'sports.sports_league_draft.picks', 'm.0v317s8'],\n", + " ['St. Louis Rams at Washington Redskins, 2009-09-20',\n", + " 'common.topic.notable_for',\n", + " 'g.1255jp1rr'],\n", + " ['m.096q4c1',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['m.0793rfw',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793k22',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793nck'],\n", + " ['m.07nn4gj',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['m.0793jzn',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.09knjsy',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09g3c7s'],\n", + " ['m.07vr58g',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Kansas City Chiefs at Washington Redskins, 2009-10-18'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793r4t'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09knjrk'],\n", + " ['m.0794brr',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['m.0793pqy',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793rcc',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['m.096q4ss',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793pdw'],\n", + " ['New Orleans Saints',\n", + " 'american_football.football_team.away_games',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['m.0794bkz',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'New Orleans Saints'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.passing',\n", + " 'm.09knjsy'],\n", + " ['m.0crtfmj',\n", + " 'sports.sports_league_participation.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07ybyjs'],\n", + " ['Team', 'rdf-schema#range', 'Professional Sports Team'],\n", + " ['m.075ftcm', 'sports.sports_team_season_record.season', '1970 NFL season'],\n", + " ['m.0794yzt',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['m.07n76kk', 'common.webpage.topic', 'Washington Redskins'],\n", + " ['Kansas City Chiefs at Washington Redskins, 2009-10-18',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07vr58v'],\n", + " ['m.0793qdy',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2008-11-16'],\n", + " ['m.09g3c9y',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.0793p2z',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2008-09-28'],\n", + " ['m.0794z0t',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Francisco 49ers'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09k0gcc'],\n", + " ['m.0793p26',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['LaRon Landry', 'sports.drafted_athlete.drafted', 'm.05bv57w'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.season',\n", + " '2008 NFL season'],\n", + " ['m.07nn4dg',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Turk Edwards', 'people.person.nationality', 'United States of America'],\n", + " ['New York Giants', 'sports.sports_team.sport', 'American football'],\n", + " ['Daniel Snyder', 'base.popstra.celebrity.friendship', 'm.065phfm'],\n", + " ['One Ring', 'fictional_universe.fictional_object.composition', 'Gold'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.away_games',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.home_team',\n", + " 'San Francisco 49ers'],\n", + " ['St. Louis Rams', 'sports.sports_team.sport', 'American football'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794bpd'],\n", + " ['Dwight Schar', 'common.topic.notable_types', 'Organization leader'],\n", + " ['m.096q4h2',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['m.0hpq5wn', 'sports.sports_team_roster.player', 'Jay Gruden'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09g3c9l'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793nyr'],\n", + " ['m.07p5wmx',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.098jkyn',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.season',\n", + " '2008 NFL season'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793xn6'],\n", + " ['m.07p5wl3',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.0793mdn',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0901v50'],\n", + " ['m.0793q8q',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.05bv57w', 'sports.sports_league_draft_pick.player', 'LaRon Landry'],\n", + " ['Curly Lambeau', 'people.person.nationality', 'United States of America'],\n", + " ['m.07ybys3',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.07922j0',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Baltimore Ravens'],\n", + " ['Mike Shanahan', 'people.person.nationality', 'United States of America'],\n", + " ['m.0794bqy',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New Orleans Saints'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793nx9'],\n", + " ['Otto Graham', 'common.topic.notable_types', 'American football player'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.season',\n", + " '2009 NFL season'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rqc'],\n", + " ['Ohio',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jl1v'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07922hn'],\n", + " ['m.0901v0l',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0793qcr',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.09g3c9l',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07922fx',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Baltimore Ravens'],\n", + " ['Mike Shanahan', 'people.person.profession', 'American football player'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qb5'],\n", + " ['m.09g3cbz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09w81c5', 'common.webpage.resource', 'NFL flattens ratings competition'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftc0'],\n", + " ['m.07t471b',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['m.07mlbn9',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.088w_qz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Atlanta Falcons'],\n", + " ['m.0901vlb',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0ggg'],\n", + " ['m.079536d',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['New York Giants',\n", + " 'american_football.football_team.home_games',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['m.07mlbsv',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['Jay Gruden',\n", + " 'american_football.football_player.position_s',\n", + " 'Quarterback'],\n", + " ['m.0793cym',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'New York Giants'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.passing',\n", + " 'm.07nn46c'],\n", + " ['New York Giants at Washington Redskins, 2008-11-30',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793qx3'],\n", + " ['m.04xh155',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt98k'],\n", + " ['m.098jkyn',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.088w_kt',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Atlanta Falcons, 2009-11-08'],\n", + " ['Vince Lombardi',\n", + " 'people.deceased_person.place_of_death',\n", + " 'Washington, D.C.'],\n", + " ['Professional Sports Team', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'common.topic.notable_for',\n", + " 'g.1255xx97v'],\n", + " ['m.0793rvz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.075ft8r', 'sports.sports_team_season_record.season', '1981 NFL season'],\n", + " ['m.0793q64',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Dwight Schar', 'people.person.education', 'm.06659xk'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'american_football.football_game.passing',\n", + " 'm.07925s3'],\n", + " ['m.098jk_s',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793ncy'],\n", + " ['m.0901t_7',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2009-11-22'],\n", + " ['m.0793nlg',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['National Football Conference',\n", + " 'organization.organization.child',\n", + " 'm.0ck2dx2'],\n", + " ['m.0793qf8',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0crt9dp', 'sports.sports_league_participation.team', 'Chicago Bears'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'american_football.football_game.passing',\n", + " 'm.07ybynn'],\n", + " ['m.09rdltm',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'San Diego Chargers'],\n", + " ['m.088w_sr',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Atlanta Falcons, 2009-11-08'],\n", + " ['m.07925sl',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Cincinnati Bengals'],\n", + " ['m.07mlbn_',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793q_k',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ft9s'],\n", + " ['m.07mlbsv',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'St. Louis Rams'],\n", + " ['m.0793xqg',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['m.0794yvg',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'San Francisco 49ers'],\n", + " ['National Football League',\n", + " 'sports.sports_league.seasons',\n", + " '2000 NFL season'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09g3ccn'],\n", + " ['White', 'base.schemastaging.visual_color_extra.srgb', 'm.010q1h63'],\n", + " ['Jason Campbell', 'people.person.profession', 'American football player'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.receiving',\n", + " 'm.096q4cz'],\n", + " ['m.075ft4s', 'sports.sports_team_season_record.season', '1994 NFL season'],\n", + " ['m.08nl7z1',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.historical_coaching_staff',\n", + " 'm.04xg975'],\n", + " ['m.0hpn6b3', 'education.education.student', 'Jay Gruden'],\n", + " ['Lud Wray', 'people.person.profession', 'American football player'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ft5_'],\n", + " ['m.0793rm_',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['m.0792c6m',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Pittsburgh Steelers'],\n", + " ['m.09rdl_m',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.062w38x', 'base.events.performance.event', 'Super Bowl XXVI'],\n", + " ['m.07p5wpg',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Tampa Bay Buccaneers'],\n", + " ['m.09g3cf0',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793pnr'],\n", + " ['Hail to the Redskins', 'common.topic.notable_types', 'Composition'],\n", + " ['m.07nn4dt',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.0793ns_',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793bcr',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0crt9dg',\n", + " 'sports.sports_league_participation.league',\n", + " 'National Football Conference'],\n", + " ['m.07925tv',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['m.07ybysg',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.09knjrx',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.0793nd8',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793nfd'],\n", + " ['m.08nl7_8',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['1937 NFL Championship Game', 'common.topic.notable_for', 'g.1258l5h8w'],\n", + " ['m.0793pvq',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07nn49k'],\n", + " ['m.07ybyp2',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['Lud Wray', 'people.person.nationality', 'United States of America'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'american_football.football_game.away_team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftg3'],\n", + " ['m.09rdm2r',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['m.08nl7wd',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0jbcqvk'],\n", + " ['m.09g3c5c',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.04xg975',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Vince Lombardi'],\n", + " ['m.0793r4t',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07p5wr5',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Tampa Bay Buccaneers'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793rlc'],\n", + " ['m.07927wp',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['m.07nn48w',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793qlw'],\n", + " ['Washington, D.C.',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'american_football.football_game.home_team',\n", + " 'Washington Redskins'],\n", + " ['NFC East', 'sports.sports_league.teams', 'm.0crtflp'],\n", + " ['m.09rdm2d',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Sports Facility', 'freebase.type_profile.kind', 'Definition'],\n", + " ['m.098jl50',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0ggt'],\n", + " ['National Football League',\n", + " 'sports.sports_league.seasons',\n", + " '2003 NFL season'],\n", + " ['Steve Spurrier', 'people.person.profession', 'American football player'],\n", + " ['m.07nn48w',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['FedExField', 'location.location.events', \"1999 FIFA Women's World Cup\"],\n", + " ['m.098jl6s',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['White', 'base.petbreeds.dog_coat_color.dog_breeds', 'Border Collie'],\n", + " ['University of Miami',\n", + " 'education.educational_institution_campus.educational_institution',\n", + " 'University of Miami'],\n", + " ['Tampa Bay Buccaneers',\n", + " 'american_football.football_team.away_games',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['1942 NFL Championship Game',\n", + " 'common.topic.notable_types',\n", + " 'Sports League Championship Event'],\n", + " ['m.07ybyrf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0792c0c',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Pittsburgh Steelers'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qgb'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793pm8'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.rushing',\n", + " 'm.08nl7yq'],\n", + " ['m.075ftdm',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.08nl7v7',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Daniel Snyder', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0793q32',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09knjp4'],\n", + " ['San Francisco 49ers', 'sports.sports_team.league', 'm.0crt9bh'],\n", + " ['m.0793qby',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07nn4c_',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.0793nfd',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.04xh13t',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793nmj',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07nn4jl'],\n", + " ['m.0793rtc',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['Washington Redskins at Dallas Cowboys, 2008-09-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793p5l'],\n", + " ['m.0793rvc',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09rdm0n'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793nm5'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09rdlvd'],\n", + " ['Super Bowl XVII', 'common.topic.notable_for', 'g.125b8t6hv'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793bc4'],\n", + " ['m.0793rvc',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['New York Giants at Washington Redskins, 2008-11-30',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793gd0'],\n", + " ['m.09knjqj',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Griffith Stadium', 'location.location.containedby', 'Washington, D.C.'],\n", + " ['m.07ybym5',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt99w'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'common.topic.notable_for',\n", + " 'g.1258c4l0h'],\n", + " ['m.09knjws',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793b9n'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0901v69'],\n", + " ['m.07p5wm5',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['m.07jqjd6',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09g3cf0'],\n", + " ['m.07p5wr5',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['m.0793n_q',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Cleveland Browns at Washington Redskins, 2008-10-19',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793ps6'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['m.07t476l',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['m.07jqjkx',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0901vdx',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2009-11-22'],\n", + " ['Tampa Bay Buccaneers at Washington Redskins, 2009-10-04',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07p5wkf'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09rdm3f'],\n", + " ['m.09knjrk',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.0793pk_',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.rushing',\n", + " 'm.08nl7qv'],\n", + " ['m.04xh13y',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rxk'],\n", + " ['m.09k0gf2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New York Giants'],\n", + " ['m.079537q',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['m.07ybyrs',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.away_games',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.07vr5hd',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Kansas City Chiefs at Washington Redskins, 2009-10-18'],\n", + " ['m.07938wz',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2008-09-28'],\n", + " ['m.07nn496',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['Carlos Rogers', 'people.person.profession', 'American football player'],\n", + " ['Dwight Schar',\n", + " 'sports.sports_team_owner.teams_owned',\n", + " 'Washington Redskins'],\n", + " ['Joe Gibbs', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rwt'],\n", + " ['m.0795339',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['St. Louis Rams at Washington Redskins, 2009-09-20',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07mlblv'],\n", + " ['m.0793qsz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Carolina Panthers',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['m.0793qpn',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07nn4dt',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['Terry Robiskie', 'people.person.gender', 'Male'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'common.topic.notable_for',\n", + " 'g.125bq2bry'],\n", + " ['m.03x9n51',\n", + " 'organization.organization_board_membership.member',\n", + " 'Dwight Schar'],\n", + " ['m.09g3c9y',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Boston Braves', 'common.topic.notable_types', 'American football team'],\n", + " ['m.098jl1h',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09k0ghh',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['m.0793b5t',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2008-11-16'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.division',\n", + " 'NFC East'],\n", + " ['m.09knjpv',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.096q4p3',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['m.07950p6',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Seattle Seahawks'],\n", + " ['m.09g3c61',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0g8s'],\n", + " ['Atlanta Falcons', 'american_football.football_team.division', 'NFC South'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.passing',\n", + " 'm.09k0g68'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0794nfh'],\n", + " ['m.0793q3j',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Pittsburgh Steelers at Washington Redskins, 2008-11-03'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793xqg'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.home_games',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['m.0793q3j',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt9c9'],\n", + " ['m.07ybylv',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.passing',\n", + " 'm.09k0g5v'],\n", + " ['m.0793q24',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07t4778',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['Burgundy', 'common.topic.article', 'm.0680md'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793mby'],\n", + " ['Super Bowl XXVI', 'common.topic.notable_for', 'g.125grx_3x'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'Washington Redskins'],\n", + " ['m.088x01b',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Atlanta Falcons, 2009-11-08'],\n", + " ['m.075ftcc',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07jqj6x'],\n", + " ['White',\n", + " 'base.ontologies.ontology_instance.equivalent_instances',\n", + " 'm.09klnhg'],\n", + " ['Pittsburgh Steelers', 'sports.sports_team.colors', 'Gold'],\n", + " ['m.0793q0v',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09k0g71'],\n", + " ['m.08nl7r5',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07ybyqr',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['Washington Redskins',\n", + " 'base.schemastaging.organization_extra.phone_number',\n", + " 'm.0px_fmz'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.historical_coaching_staff',\n", + " 'm.04xh13f'],\n", + " ['Daniel Snyder', 'film.producer.films_executive_produced', '23 Blast'],\n", + " ['m.096q4h2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.0793pb4',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07jqjg9',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.075ftg3',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt9dg'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'St. Louis Rams'],\n", + " ['m.05bv57w', 'sports.sports_league_draft_pick.draft', '2007 NFL draft'],\n", + " ['m.088w_nb',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Atlanta Falcons, 2009-11-08'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07nn47v'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793pcg'],\n", + " ['m.098jl0g',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['m.0h_2dl_', 'sports.sports_team_roster.position', 'Quarterback'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09rdm0_'],\n", + " ['m.0793rcy',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Cincinnati Bengals, 2008-12-14'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0901v0l'],\n", + " ['m.088w_xr',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Atlanta Falcons, 2009-11-08'],\n", + " ['m.0793b7y',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09rdm1p'],\n", + " ['NFC East', 'sports.sports_league.sport', 'American football'],\n", + " ['m.07927rj',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793b5t'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.season',\n", + " '2009 NFL season'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793pz8'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.away_team',\n", + " 'Washington Redskins'],\n", + " ['Bill McPeak', 'people.person.gender', 'Male'],\n", + " ['m.0svstws', 'education.education.student', 'Jay Gruden'],\n", + " ['Washington Redskins',\n", + " 'base.schemastaging.organization_extra.phone_number',\n", + " 'm.0px_kgz'],\n", + " ['m.07jqj2r',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['m.07vr5js',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Kansas City Chiefs'],\n", + " ['m.0793nm5',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07nn46t'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'common.topic.notable_for',\n", + " 'g.1257vp6v2'],\n", + " ['Chris Thompson', 'people.person.profession', 'American football player'],\n", + " ['m.0901tzf',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.07953bf',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'St. Louis Rams'],\n", + " ['m.0xnjm9z', 'business.employment_tenure.company', 'Washington Redskins'],\n", + " ['m.0793r0p',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793qq2'],\n", + " ['m.0793rsx',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rcy'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793k2w'],\n", + " ['m.07925vz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Cincinnati Bengals'],\n", + " ['m.0793nl3',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09g3c4b',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Ohio'],\n", + " ['Mike Shanahan', 'common.topic.notable_types', 'American football player'],\n", + " ['m.075ftnb',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.075ftlq',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['m.07938vp',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.098jkzf',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['m.07p5wnl',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['m.0px_kgz',\n", + " 'base.schemastaging.phone_sandbox.service_language',\n", + " 'English Language'],\n", + " ['m.0793q5l',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Pittsburgh Steelers at Washington Redskins, 2008-11-03'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793bbm'],\n", + " ['National Football Conference',\n", + " 'common.topic.notable_types',\n", + " 'American football conference'],\n", + " ['Cleveland Browns',\n", + " 'american_football.football_team.away_games',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.historical_coaching_staff',\n", + " 'm.04xh149'],\n", + " ['Super Bowl', 'sports.sports_championship.events', 'Super Bowl XVII'],\n", + " ['m.075fxp4', 'sports.sports_team_season_record.team', 'Boston Redskins'],\n", + " ['Tampa Bay Buccaneers',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['Oakland Raiders',\n", + " 'american_football.football_team.home_games',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.0crt9d0',\n", + " 'sports.sports_league_participation.league',\n", + " 'National Football Conference'],\n", + " ['m.0793qhs',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2008-11-16'],\n", + " ['m.07jqjb4',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['m.0794z0t',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['m.075fxpd', 'sports.sports_team_season_record.team', 'Boston Redskins'],\n", + " ['m.0793nn0',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07nn485',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['FedExField', 'location.location.containedby', 'Landover'],\n", + " ['m.0793nks',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['m.0792c3v',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Pittsburgh Steelers'],\n", + " ['m.04xh14k',\n", + " 'american_football.football_historical_coach_position.position',\n", + " 'Head coach'],\n", + " ['Super Bowl XVII', 'common.topic.article', 'm.076r_'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jl0g'],\n", + " ['m.08nl7r5',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['m.096q4rw',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.05bv748', 'sports.sports_league_draft_pick.draft', '2005 NFL draft'],\n", + " ['Washington Redskins', 'sports.sports_team.coaches', 'm.0z_fbv7'],\n", + " ['m.098jl5r',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.receiving',\n", + " 'm.088x01b'],\n", + " ['m.0793nv2',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793nlt',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Jim Zorn', 'people.person.nationality', 'United States of America'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.away_team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0793q_4',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2008-11-30'],\n", + " ['m.0793py3',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.08nl7y9',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Denver Broncos'],\n", + " ['m.065pkmr', 'base.popstra.friendship.participant', 'Daniel Snyder'],\n", + " ['m.0793rgn',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ft6l'],\n", + " ['m.07nn4hk',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793mcp',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Joe Gibbs', 'people.person.gender', 'Male'],\n", + " ['m.0793k2w',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2008-10-05'],\n", + " ['m.0794bqy',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['Tampa Bay Buccaneers at Washington Redskins, 2009-10-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07p5ws6'],\n", + " ['New York Giants',\n", + " 'american_football.football_team.home_games',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['Robert Rothman', 'people.person.gender', 'Male'],\n", + " ['m.09rdlwr',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['m.096q4tb',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.075ftb0',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftdc'],\n", + " ['Washington Redskins at Seattle Seahawks, 2008-11-23',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07950rs'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793rky'],\n", + " ['National Football Conference', 'sports.sports_league.teams', 'm.0crt9dp'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07t471b'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0g82'],\n", + " ['m.0793nlt',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2008-09-14'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.home_games',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'San Francisco 49ers'],\n", + " ['m.098jl14',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793mfs'],\n", + " ['Kansas City Chiefs at Washington Redskins, 2009-10-18',\n", + " 'american_football.football_game.season',\n", + " '2009 NFL season'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793xrz'],\n", + " ['m.0crt99h',\n", + " 'sports.sports_league_participation.league',\n", + " 'National Football Conference'],\n", + " ['FedExField', 'architecture.structure.owner', 'm.05pt0g9'],\n", + " ['m.07950gm',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Seattle Seahawks'],\n", + " ['Cleveland Browns at Washington Redskins, 2008-10-19',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07927s0'],\n", + " ['Braves Field', 'sports.sports_facility.home_venue_for', 'm.0n5zj2z'],\n", + " ['m.09k0g7r',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['m.07p5wkf',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.passing',\n", + " 'm.08nl7ny'],\n", + " ['m.0crt988',\n", + " 'sports.sports_league_participation.team',\n", + " 'Green Bay Packers'],\n", + " ['Herman Ball',\n", + " 'american_football.football_coach.coaching_history',\n", + " 'm.04xh139'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['m.0793rlc',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jk_s'],\n", + " ['m.098jl03',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['NFC North', 'organization.organization.parent', 'm.0ck2dx8'],\n", + " ['Super Bowl XXVI',\n", + " 'base.events.performance_event.performance',\n", + " 'm.062w39b'],\n", + " ['National Football Conference',\n", + " 'common.topic.image',\n", + " '200px-National_Football_Conference_logo_svg.png'],\n", + " ['m.0ztj_hm',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Mike Shanahan'],\n", + " ['m.07vr5c1',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0crt572', 'sports.sports_league_participation.team', 'Boston Redskins'],\n", + " ['Ohio',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['m.0793gfp',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New York Giants'],\n", + " ['m.07938t0',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.07t470_',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Carolina Panthers'],\n", + " ['m.0793r47',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Baltimore Ravens, 2008-12-07'],\n", + " ['Silver Spring', 'location.hud_county_place.place', 'Silver Spring'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0901v1q'],\n", + " ['m.04xh12k',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Eddie Casey'],\n", + " ['m.09k0gfs',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['Super Bowl XXII', 'common.topic.notable_types', 'Super bowl'],\n", + " ['Valkyrie', 'film.film.language', 'English Language'],\n", + " ['m.07nn47h',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0794yvv'],\n", + " ['m.0793pm8',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['2005 NFL season',\n", + " 'sports.sports_league_season.league',\n", + " 'National Football League'],\n", + " ['St. Louis Rams', 'american_football.football_team.division', 'NFC West'],\n", + " ['m.09rdm1p',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['Dwight Schar', 'common.topic.notable_for', 'g.1255mqhqh'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rrc'],\n", + " ['m.075ftjg', 'sports.sports_team_season_record.season', '1951 NFL season'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.season',\n", + " '2008 NFL season'],\n", + " ['Daniel Snyder',\n", + " 'business.board_member.organization_board_memberships',\n", + " 'm.05lkgyd'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07t477z'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07jqjjd'],\n", + " ['m.0793nlg',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins', 'common.topic.image', 'Washington Redskins Helmet'],\n", + " ['m.0793pz8',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2008-10-26'],\n", + " ['m.0793rmc',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['George Preston Marshall', 'common.topic.notable_for', 'g.1259gdhdj'],\n", + " ['m.04xh139',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793xq1'],\n", + " ['m.08nl7qv',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Otto Graham',\n", + " 'american_football.football_coach.coaching_history',\n", + " 'm.04xh141'],\n", + " ['m.0793qvk',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Carlos Rogers', 'people.person.gender', 'Male'],\n", + " ['m.096q4bc',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794bqy'],\n", + " ['m.09rdlxf',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['m.0793qhs',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['New York Giants', 'sports.sports_team.league', 'm.0crtfm0'],\n", + " ['Washington Redskins at Oakland Raiders, 2009-12-13',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09g3c97'],\n", + " ['Daniel Snyder', 'film.producer.films_executive_produced', 'Valkyrie'],\n", + " ['m.07nn4j7',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.passing',\n", + " 'm.0793nv2'],\n", + " ['m.075ft98',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.08nl7rx',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09knjw2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0794yzd',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Francisco 49ers'],\n", + " ['m.0793pk_',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['Washington Redskins',\n", + " 'organization.organization.founders',\n", + " 'George Preston Marshall'],\n", + " ['m.07nn496',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09knjr6'],\n", + " ['Jay Gruden', 'sports.pro_athlete.teams', 'm.0hpq5wn'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'common.topic.notable_for',\n", + " 'g.1259m4sn3'],\n", + " ['m.0793xpk',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['Washington Redskins at New York Giants, 2009-09-13',\n", + " 'american_football.football_game.passing',\n", + " 'm.07jqjcd'],\n", + " ['m.0794njj',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07922k4'],\n", + " ['Daniel Snyder', 'people.person.place_of_birth', 'Silver Spring'],\n", + " ['m.0901v2s',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['NFL flattens ratings competition',\n", + " 'common.resource.annotations',\n", + " 'm.09w81c5'],\n", + " ['m.0793b5t',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0793qs7',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0crt97_',\n", + " 'sports.sports_league_participation.team',\n", + " 'Minnesota Vikings'],\n", + " ['m.075ft89',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.07mlbl5',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['m.07t4767',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Carolina Panthers, 2009-10-11'],\n", + " ['m.0793r8b',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.05bv768', 'sports.sports_league_draft_pick.player', 'Jason Campbell'],\n", + " ['m.07mlbvf',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['Super Bowl', 'time.recurring_event.instances', 'Super Bowl XXII'],\n", + " ['m.07jqj64',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2009-09-13'],\n", + " ['m.07mlbx0',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'St. Louis Rams'],\n", + " ['m.03x2_kx', 'business.employment_tenure.company', 'NVR, Inc.'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09k0g7r'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793rsx'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07nn4hx'],\n", + " ['m.0793qq2',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['m.096q4cz',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.07nn49k',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['Chicago Bears', 'sports.sports_team.league', 'm.0crt9dp'],\n", + " ['m.09knjr6',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.04xh14t',\n", + " 'american_football.football_historical_coach_position.position',\n", + " 'Head coach'],\n", + " ['m.07t4731',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Carolina Panthers'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftcc'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.rushing',\n", + " 'm.088w_fz'],\n", + " ['m.0793nc3',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['FedExField', 'location.location.geolocation', 'm.02_tfms'],\n", + " ['m.0793rpk',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07vr5b6',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07t474w',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793rtc',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rvz'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.home_team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.07ng0h9',\n", + " 'base.ontologies.ontology_instance_mapping.freebase_topic',\n", + " 'Washington Redskins'],\n", + " ['m.096q4ss',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['m.096q46m',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793r47'],\n", + " ['Cleveland Browns',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['m.08nl7z1',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Denver Broncos'],\n", + " ['Washington Redskins at Atlanta Falcons, 2009-11-08',\n", + " 'american_football.football_game.passing',\n", + " 'm.088w_dh'],\n", + " ['m.09k0gbh',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['m.0793nwf',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Arizona Cardinals at Washington Redskins, 2008-09-21'],\n", + " ['m.0crt9dg', 'sports.sports_league_participation.team', 'Dallas Cowboys'],\n", + " ['m.09g3cf0',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Daniel Snyder',\n", + " 'business.board_member.organization_board_memberships',\n", + " 'm.05lkgys'],\n", + " ['m.0crt99w', 'sports.sports_league_participation.team', 'New York Giants'],\n", + " ['Norv Turner',\n", + " 'american_football.football_coach.coaching_history',\n", + " 'm.04xh14t'],\n", + " ['m.0793qxx',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.04xh149',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['m.09rdlyt',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793q24'],\n", + " ['m.0793n_1',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793ndm'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793k22'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793c_t'],\n", + " ['Seattle Seahawks',\n", + " 'american_football.football_team.home_games',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['m.0793nck',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['m.0793nyr',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793r57',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09g3c83',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.receiving',\n", + " 'm.096q4fv'],\n", + " ['m.09g3c36',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Oakland Raiders, 2009-12-13'],\n", + " ['m.07t477m',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.098jl25',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09k0g68',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'New York Giants at Washington Redskins, 2009-12-21'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2009-11-29',\n", + " 'american_football.football_game.rushing',\n", + " 'm.096q4c1'],\n", + " ['m.05lkg14', 'business.employment_tenure.person', 'Daniel Snyder'],\n", + " ['m.079533s',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['Tampa Bay Buccaneers at Washington Redskins, 2009-10-04',\n", + " 'common.topic.notable_for',\n", + " 'g.1255cmykp'],\n", + " ['m.0793qqm',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Dallas Cowboys', 'sports.sports_team.sport', 'American football'],\n", + " ['m.0793nc3',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['Dallas Cowboys', 'sports.sports_team.colors', 'White'],\n", + " ['Dallas Cowboys at Washington Redskins, 2008-11-16',\n", + " 'american_football.football_game.passing',\n", + " 'm.0793qby'],\n", + " ['m.075ft6v',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.0792c0t',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Pittsburgh Steelers at Washington Redskins, 2008-11-03'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793mhq'],\n", + " ['m.09g3c36',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Oakland Raiders'],\n", + " ['Super Bowl XXVI',\n", + " 'sports.sports_championship_event.season',\n", + " '1991 NFL season'],\n", + " ['White',\n", + " 'base.roses.rose_color.roses_of_this_color',\n", + " \"Rosa 'Frau Karl Druschki'\"],\n", + " ['m.07p5wn7',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['National Football League',\n", + " 'sports.sports_league.seasons',\n", + " '2008 NFL season'],\n", + " ['m.0crt9b6', 'sports.sports_league_participation.team', 'Detroit Lions'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07nn4f4'],\n", + " ['Professional Sports Team', 'type.type.domain', 'Sports'],\n", + " ['m.07938wj',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['Jay Gruden', 'people.person.profession', 'American football player'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'american_football.football_game.season',\n", + " '2008 NFL season'],\n", + " ['m.0793xvr',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2008-10-26'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07925vk'],\n", + " ['m.075ft1j',\n", + " 'sports.sports_team_season_record.team',\n", + " 'Washington Redskins'],\n", + " ['m.0794yts',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at San Francisco 49ers, 2008-12-28'],\n", + " ['American football player',\n", + " 'freebase.equivalent_topic.equivalent_type',\n", + " 'American football player'],\n", + " ['m.05lmmc8',\n", + " 'organization.organization_board_membership.member',\n", + " 'Daniel Snyder'],\n", + " ['m.09knjqj',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.0793nlg',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['1942 NFL Championship Game',\n", + " 'sports.sports_championship_event.runner_up',\n", + " 'Chicago Bears'],\n", + " ['m.0793cxn',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'New York Giants'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.09rdl_z'],\n", + " ['m.098jl5c',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['m.0793ps6',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793jxw'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.passing',\n", + " 'm.0901tw7'],\n", + " ['m.09k0g9t',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.09rdm1p',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.07nn4c_',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'Seattle Seahawks'],\n", + " ['m.096q4rw',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Philadelphia Eagles, 2009-11-29'],\n", + " ['Washington Redskins', 'common.topic.webpage', 'm.0422763'],\n", + " ['Chicago Bears', 'sports.sports_team.colors', 'White'],\n", + " ['m.09rdm0n',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at San Diego Chargers, 2010-01-03'],\n", + " ['m.0ck2dx8',\n", + " 'organization.organization_relationship.parent',\n", + " 'National Football Conference'],\n", + " ['m.09knjwf',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.0793mdn',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['Washington Redskins at San Diego Chargers, 2010-01-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09rdm1b'],\n", + " ['National Football Conference', 'common.topic.article', 'm.01ddr1'],\n", + " ['National Football League',\n", + " 'sports.sports_league.seasons',\n", + " '2001 NFL season'],\n", + " ['m.07mlbqy',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Cleveland Browns at Washington Redskins, 2008-10-19',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793pr8'],\n", + " ['m.098jk_f',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'New Orleans Saints at Washington Redskins, 2009-12-06'],\n", + " ['Tampa Bay Buccaneers at Washington Redskins, 2009-10-04',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07p5wpg'],\n", + " ['m.096q4kd',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['2009 NFL season',\n", + " 'sports.sports_league_season.league',\n", + " 'National Football League'],\n", + " ['m.09rdm0_',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at Carolina Panthers, 2009-10-11',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07t4767'],\n", + " ['Team', 'type.property.expected_type', 'Professional Sports Team'],\n", + " ['m.0793nd8',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at New York Giants, 2008-09-04',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793cy2'],\n", + " ['m.09rdly3',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'San Diego Chargers'],\n", + " ['Washington Redskins',\n", + " 'common.topic.notable_types',\n", + " 'Professional Sports Team'],\n", + " ['m.0793pnr',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2008-10-12'],\n", + " ['m.0793ndm',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['Cleveland Browns at Washington Redskins, 2008-10-19',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07927v4'],\n", + " ['m.09knjp4',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['m.07mlbwn',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['m.08nl7tr',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.rushing',\n", + " 'm.08nl7qh'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07922gf'],\n", + " ['Kansas City Chiefs',\n", + " 'common.topic.notable_types',\n", + " 'American football team'],\n", + " ['Kansas City Chiefs at Washington Redskins, 2009-10-18',\n", + " 'american_football.football_game.passing',\n", + " 'm.07vr577'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'Arizona Cardinals'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794nhb'],\n", + " ['Minnesota Vikings',\n", + " 'sports.sports_team.arena_stadium',\n", + " 'Hubert H. Humphrey Metrodome'],\n", + " ['Tampa Bay Buccaneers', 'sports.sports_team.league', 'm.0crt98w'],\n", + " ['St. Louis Rams at Washington Redskins, 2008-10-12',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0795357'],\n", + " ['New Orleans Saints at Washington Redskins, 2008-09-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793npq'],\n", + " ['NFC South', 'sports.sports_league.sport', 'American football'],\n", + " ['Dudley DeGroot', 'people.person.gender', 'Male'],\n", + " ['m.0h_2dl_', 'sports.sports_team_roster.player', 'Jay Gruden'],\n", + " ['m.08nl7xm',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Denver Broncos at Washington Redskins, 2009-11-15'],\n", + " ['Daniel Snyder', 'organization.organization.headquarters', 'm.09sbjsq'],\n", + " ['Dallas Cowboys at Washington Redskins, 2009-12-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09knjw2'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0792c31'],\n", + " ['m.0792c2p',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Pittsburgh Steelers'],\n", + " ['NFC East',\n", + " 'american_football.football_division.conference',\n", + " 'National Football Conference'],\n", + " ['m.0793qlw',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.04xh14t',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference',\n", + " 'organization.organization.parent',\n", + " 'm.04kjpjb'],\n", + " ['1937 NFL Championship Game',\n", + " 'sports.sports_championship_event.runner_up',\n", + " 'Chicago Bears'],\n", + " ['m.09rdm09',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins',\n", + " 'american_football.football_team.home_games',\n", + " 'Pittsburgh Steelers at Washington Redskins, 2008-11-03'],\n", + " ['m.09knjph',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.07nn485',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Detroit Lions, 2009-09-27'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftlq'],\n", + " ['m.0793pdw',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['m.0793b69',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2008-11-16'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07nn4gj'],\n", + " ['m.096q4gn',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Philadelphia Eagles'],\n", + " ['m.04xh12y',\n", + " 'american_football.football_historical_coach_position.team',\n", + " 'Washington Redskins'],\n", + " ['National Football Conference',\n", + " 'american_football.football_conference.teams',\n", + " 'Detroit Lions'],\n", + " ['m.07vr5hr',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Kansas City Chiefs at Washington Redskins, 2009-10-18'],\n", + " ['New Orleans Saints at Washington Redskins, 2009-12-06',\n", + " 'american_football.football_game.receiving',\n", + " 'm.098jl03'],\n", + " ['Daniel Snyder', 'people.person.profession', 'Film Producer'],\n", + " ['m.07ybyt4',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2009-10-26'],\n", + " ['Arizona Cardinals',\n", + " 'american_football.football_team.division',\n", + " 'NFC West'],\n", + " ['m.09k0ggg',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'New York Giants'],\n", + " ['m.07950r2',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Seattle Seahawks'],\n", + " ['San Francisco 49ers',\n", + " 'american_football.football_team.conference',\n", + " 'National Football Conference'],\n", + " ['Washington Redskins at Cincinnati Bengals, 2008-12-14',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793rdb'],\n", + " ['m.0crt97k',\n", + " 'sports.sports_league_participation.league',\n", + " 'National Football Conference'],\n", + " ['m.04xh12y',\n", + " 'american_football.football_historical_coach_position.coach',\n", + " 'Dudley DeGroot'],\n", + " ['Denver Broncos at Washington Redskins, 2009-11-15',\n", + " 'american_football.football_game.season',\n", + " '2009 NFL season'],\n", + " ['m.07938tt',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at Dallas Cowboys, 2008-09-28'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.07922jc'],\n", + " ['m.07mlbjn',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'St. Louis Rams at Washington Redskins, 2009-09-20'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07nn474'],\n", + " ['m.062w39b', 'base.events.performance.event', 'Super Bowl XXVI'],\n", + " ['Washington Redskins at Baltimore Ravens, 2008-12-07',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793r6q'],\n", + " ['Arizona Cardinals at Washington Redskins, 2008-09-21',\n", + " 'american_football.football_game.passing',\n", + " 'm.0794ndf'],\n", + " ['m.09knjxt',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Dallas Cowboys'],\n", + " ['m.0793rbh',\n", + " 'american_football.game_rushing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['Pittsburgh Steelers at Washington Redskins, 2008-11-03',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793q64'],\n", + " ['m.0793ng5',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['2000 NFL season',\n", + " 'sports.sports_league_season.league',\n", + " 'National Football League'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ft8r'],\n", + " ['Dwight Schar', 'base.schemastaging.person_extra.net_worth', 'm.0n4phph'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2008-12-21',\n", + " 'common.topic.notable_for',\n", + " 'g.1256dfq_c'],\n", + " ['Super Bowl XXVI',\n", + " 'base.events.performance_event.performance',\n", + " 'm.062w38x'],\n", + " ['m.07950r2',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Washington Redskins at Seattle Seahawks, 2008-11-23'],\n", + " ['New York Giants at Washington Redskins, 2009-12-21',\n", + " 'american_football.football_game.receiving',\n", + " 'm.09k0gg3'],\n", + " ['m.0793cy2',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Washington Redskins at New York Giants, 2008-09-04'],\n", + " ['m.09rdl_z',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Owner(s)', 'owl#inverseOf', 'Teams Owned'],\n", + " ['Washington Redskins at Dallas Cowboys, 2009-11-22',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0901vcz'],\n", + " ['New York Giants at Washington Redskins, 2008-11-30',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793gbz'],\n", + " ['Washington Redskins at Detroit Lions, 2009-09-27',\n", + " 'common.topic.notable_types',\n", + " 'NFL Game'],\n", + " ['Washington Redskins', 'sports.sports_team.season_record', 'm.075ftb0'],\n", + " ['m.05lmmgj',\n", + " 'organization.organization_board_membership.organization',\n", + " 'Washington Redskins'],\n", + " ['m.0793nv2',\n", + " 'american_football.game_passing_statistics.player',\n", + " 'Jason Campbell'],\n", + " ['m.0793md5',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Philadelphia Eagles at Washington Redskins, 2008-12-21'],\n", + " ['m.07927x1',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Cleveland Browns'],\n", + " ['Philadelphia Eagles at Washington Redskins, 2009-10-26',\n", + " 'american_football.football_game.rushing',\n", + " 'm.07ybypr'],\n", + " ['m.09knjtq',\n", + " 'american_football.game_rushing_statistics.game',\n", + " 'Dallas Cowboys at Washington Redskins, 2009-12-27'],\n", + " ['NFC North',\n", + " 'american_football.football_division.teams',\n", + " 'Minnesota Vikings'],\n", + " ['Chicago Bears', 'american_football.football_team.division', 'NFC North'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.passing',\n", + " 'm.0793p8m'],\n", + " ['m.07nn4c9',\n", + " 'american_football.game_receiving_statistics.team',\n", + " 'Detroit Lions'],\n", + " ['m.0793py3',\n", + " 'american_football.game_passing_statistics.team',\n", + " 'Washington Redskins'],\n", + " ['Washington Redskins at San Francisco 49ers, 2008-12-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0794yws'],\n", + " ['m.07p5wjh',\n", + " 'american_football.game_passing_statistics.game',\n", + " 'Tampa Bay Buccaneers at Washington Redskins, 2009-10-04'],\n", + " ['m.07922h7',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'Baltimore Ravens'],\n", + " ['Washington Redskins at Dallas Cowboys, 2008-09-28',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793p6t'],\n", + " ['m.075ft89', 'sports.sports_team_season_record.season', '1982 NFL season'],\n", + " ['m.0793pws',\n", + " 'american_football.game_receiving_statistics.game',\n", + " 'Cleveland Browns at Washington Redskins, 2008-10-19'],\n", + " ['m.05bv748', 'sports.sports_league_draft_pick.player', 'Carlos Rogers'],\n", + " ['m.098jl4b',\n", + " 'american_football.game_rushing_statistics.team',\n", + " 'New Orleans Saints'],\n", + " ['Gold',\n", + " 'fictional_universe.fictional_substance.fictional_object',\n", + " 'One Ring'],\n", + " ['Washington Redskins at Philadelphia Eagles, 2008-10-05',\n", + " 'american_football.football_game.rushing',\n", + " 'm.0793jyz'],\n", + " ['Atlanta Falcons',\n", + " 'american_football.football_team.conference',\n", + " 'National Football Conference'],\n", + " ['Washington Redskins at Detroit Lions, 2008-10-26',\n", + " 'american_football.football_game.receiving',\n", + " 'm.0793pzt'],\n", + " ['Dick Todd', 'people.person.profession', 'American football player'],\n", + " ...],\n", + " [['m.02_1x1x',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Emma Goldman', 'common.topic.notable_types', 'Author'],\n", + " ['Cimetière parisien de Bagneux',\n", + " 'people.place_of_interment.interred_here',\n", + " 'Oscar Wilde'],\n", + " ['m.0j5g7_q', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Biographer', 'people.profession.specialization_of', 'Writer'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'The Importance of Being Earnest'],\n", + " [\"Events in 2011 and Forthcoming Book Strengthen James Baldwin's Legacy\",\n", + " 'book.written_work.school_or_movement',\n", + " 'Biography'],\n", + " ['When William Came', 'book.written_work.subjects', 'Literature'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The \\\\\"Canterville Ghost\\\\\"'],\n", + " ['Beaumont-Hamel', 'common.topic.article', 'm.04tr23'],\n", + " ['Oscar Wilde',\n", + " 'influence.influence_node.influenced_by',\n", + " 'William Shakespeare'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['Stephen Fry', 'people.person.profession', 'Poet'],\n", + " ['Author',\n", + " 'tv.tv_subject.tv_episodes',\n", + " 'Leisha Kelly – author- \\\\\"House on Malcolm Street\\\\\"'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010fm15_'],\n", + " ['Librettist', 'people.profession.specialization_of', 'Writer'],\n", + " ['m.09y2zb5', 'common.webpage.topic', 'Oscar Wilde'],\n", + " ['Oscar Wilde', 'media_common.dedicator.dedications', 'm.04_m4gz'],\n", + " ['Oscar Wilde', 'common.topic.notable_types', 'Author'],\n", + " ['When William Came', 'book.book.editions', 'When William Came'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Transient ischemic attack'],\n", + " ['Stephen Fry', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Playwright', 'people.profession.specialization_of', 'Writer'],\n", + " ['Charles Augustus Munro', 'people.person.children', 'Saki'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'His works such as the Picture of Dorian Grey, The Importance of Being Earnest, and the Ballad of Reading Gaol continue to be relevant today.'],\n", + " ['Dante Alighieri', 'people.person.gender', 'Male'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Alejandro Dolina'],\n", + " ['Aristotle', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Plato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Honoré de Balzac', 'influence.influence_node.influenced', 'Oscar Wilde'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gout'],\n", + " ['Somme',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Picardy'],\n", + " ['m.0w16nb3', 'education.education.student', 'Oscar Wilde'],\n", + " ['Sredni Vashtar, and other stories',\n", + " 'common.topic.notable_for',\n", + " 'g.125bjf00l'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Barrett's esophagus\"],\n", + " ['Oscar Wilde', 'people.person.sibling_s', 'm.0r5qxs_'],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.parent_disease',\n", + " 'Cardiovascular disease'],\n", + " ['André Gide', 'people.person.profession', 'Novelist'],\n", + " ['Oscar Wilde', 'film.film_story_contributor.film_story_credits', 'The Fan'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Prostate cancer'],\n", + " ['Society of Authors', 'people.person.profession', 'Writer'],\n", + " ['William Shakespeare',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date of death'],\n", + " ['Oscar Wilde', 'people.person.education', 'm.02wngvx'],\n", + " ['Peter Kropotkin', 'influence.influence_node.influenced', 'Oscar Wilde'],\n", + " ['Author', 'people.profession.specialization_of', 'Writer'],\n", + " ['Walt Whitman', 'influence.influence_node.influenced', 'Jorge Luis Borges'],\n", + " ['Beasts and Super-Beasts',\n", + " 'book.written_work.previous_in_series',\n", + " 'When William Came'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Wilde wrote Salomé in French and the play was illustrated by English artist and fellow aesthete, Beardsley, who would go on to draw other less than flattering character sketches of Wilde.'],\n", + " ['Aristotle', 'influence.influence_node.influenced_by', 'Plato'],\n", + " ['Saki', 'book.author.works_written', 'The Unbearable Bassington'],\n", + " ['Saki', 'book.author.works_written', 'Cuentos Indiscretos'],\n", + " ['André Gide', 'people.person.profession', 'Writer'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'peter alexeyevich kropotkin a supporter was oscar wilde'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['Jorge Luis Borges', 'people.person.profession', 'Novelist'],\n", + " ['m.0pbrrqb', 'film.performance.film', 'Logs [Window]'],\n", + " ['Stephen Fry',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Garth Marenghi'],\n", + " ['Columnist', 'people.profession.specialization_of', 'Writer'],\n", + " ['Picardy',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'France'],\n", + " ['Oscar Wilde', 'book.author.book_editions_published', 'Canterville ghost.'],\n", + " ['Epilepsy', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Writer', 'people.profession.specializations', 'P.R.O, Writer'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'william butler yeats a pal of oscar wilde'],\n", + " ['heart attack',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Stephen Fry', 'people.person.profession', 'Author'],\n", + " ['Stephen Fry', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Sredni Vashtar', 'film.film.starring', 'm.0nf33dh'],\n", + " ['Author', 'common.topic.subject_of', 'Jeremy Yablan'],\n", + " ['Conrad the Wise', 'film.film.film_casting_director', 'Kirsten Dewolfe'],\n", + " ['The Toys of Peace, and Other Papers',\n", + " 'book.book.editions',\n", + " 'The Toys of Peace, and Other Papers'],\n", + " ['Mary Frances Mercer', 'common.topic.notable_types', 'Deceased Person'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.07n73w_'],\n", + " ['m.0p7b4yp', 'film.performance.film', 'Logs [Window]'],\n", + " ['Myanmar',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Rakhine State'],\n", + " ['Cyrano de Bergerac',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Playwright'],\n", + " ['m.0pbrsb2', 'film.performance.film', 'Logs [Window]'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Bathilda Bagshot'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " 'Scandal is merely the compassionate allowance which the gay make to the humdrum. Think how many blameless lives are brightened by the blazing indiscretions of other people.'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0j7v__g'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.011nj8nh'],\n", + " [\"Parkinson's disease\",\n", + " 'medicine.disease.symptoms',\n", + " 'Seborrheic dermatitis'],\n", + " ['m.0nf30nj', 'people.marriage.spouse', 'Charles Augustus Munro'],\n", + " ['Writer', 'people.profession.specializations', 'Dramaturgist'],\n", + " ['m.02_98hd',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'Canterville Ghost (Swc 2051)'],\n", + " ['He spends his life explaining from his pulpit that the glory of Christianity consists in the fact that though it is not true it has been found necessary to invent it.',\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['Olivia Wilde', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0r5qx50', 'people.sibling_relationship.sibling', 'Oscar Wilde'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'oscar wilde wrote sonnets to ellen terry'],\n", + " [\"Events in 2011 and Forthcoming Book Strengthen James Baldwin's Legacy\",\n", + " 'book.written_work.subjects',\n", + " 'Author'],\n", + " ['United Kingdom', 'location.country.first_level_divisions', 'England'],\n", + " ['Writer', 'people.profession.specializations', 'Graphic Novelist'],\n", + " ['m.0j5g7_q', 'film.performance.actor', 'Darryl Bosa'],\n", + " ['m.0b6rfk0', 'common.webpage.category', 'Tag'],\n", + " ['Saki', 'book.author.works_written', 'The Complete Saki'],\n", + " ['m.0cjm4sb',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Reginald in Russia and Other Sketches',\n", + " 'common.topic.notable_types',\n", + " 'Book'],\n", + " ['Harvey Bernhard',\n", + " 'film.producer.films_executive_produced',\n", + " 'Sredni Vashtar'],\n", + " ['Literature',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '“Sociologically, politically, psychologically, spiritually, it was never enough for him [James Baldwin] to categorize himself as one thing or the other: not just black, not just sexual, not just American, nor even just as a world-class literary artist. He embraced the whole of life the way the sun’s gravitational passion embraces everything from the smallest wandering comet to the largest looming planet. He both confronted and cultivated creative vision with a drive, passion, and brilliance that few have matched, and simply being able to watch his genius sparkle from one sentence to the next could generate both awe and revelation.”'],\n", + " ['Male', 'common.topic.article', 'm.05zpq8'],\n", + " ['MA$TADON', 'people.person.gender', 'Male'],\n", + " ['Samuel Beckett', 'influence.influence_node.influenced', 'Tom Stoppard'],\n", + " [\"He's simply got the instinct for being unhappy highly developed.\",\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " [\"He's simply got the instinct for being unhappy highly developed.\",\n", + " 'media_common.quotation.subjects',\n", + " 'Happiness'],\n", + " ['Writer', 'people.profession.specializations', 'Doxographer'],\n", + " ['Victor Hugo', 'people.deceased_person.place_of_death', 'Paris'],\n", + " ['Author', 'type.type.expected_by', 'Author/editor'],\n", + " ['Author', 'type.type.expected_by', 'Authors'],\n", + " ['When William Came', 'book.book_edition.isbn', '9781406800241'],\n", + " ['Beasts and Super-Beasts', 'common.topic.article', 'm.0f0y74'],\n", + " ['The Unbearable Saki: The Work of H. H. Munro',\n", + " 'book.written_work.subjects',\n", + " 'Saki'],\n", + " ['Max Beerbohm', 'common.topic.notable_types', 'Author'],\n", + " ['Conrad the Wise', 'film.film.cinematography', 'Jorma Kantola'],\n", + " ['Journalist', 'people.profession.corresponding_type', 'Author'],\n", + " ['m.0j88xw7',\n", + " 'business.company_product_relationship.consumer_product',\n", + " 'When William Came (Large Print)'],\n", + " ['Emma Goldman', 'people.person.profession', 'Writer'],\n", + " ['Aristotle', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Dave Eggers'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['When people grow gradually rich their requirements and standard of living expand in proportion, while their present-giving instincts often remain in the undeveloped condition of their earlier days. Something showy and not-too-expensive in a shop is their only conception of the ideal gift.',\n", + " 'media_common.quotation.subjects',\n", + " 'Giving'],\n", + " ['Saki', 'people.person.sibling_s', 'm.0nf30x3'],\n", + " ['When William Came', 'book.book_edition.book', 'When William Came'],\n", + " ['Conrad the Wise', 'film.film.written_by', 'Alan Miller'],\n", + " ['Victor Hugo', 'influence.influence_node.influenced', 'Oscar Wilde'],\n", + " ['Date of burial', 'type.property.schema', 'Deceased Person'],\n", + " ['Oscar Wilde', 'people.person.sibling_s', 'm.0m_2scq'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " \"Children with Hyacinth's temperament don't know better as they grow older; they merely know more.\"],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville Ghost (English Bookmarks)'],\n", + " ['m.0p7b4yx', 'film.performance.film', 'Logs [Window]'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Dan Humphrey'],\n", + " ['m.0gyj7cx', 'base.popstra.restaurant_choice.diner', 'Oscar Wilde'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney Stone'],\n", + " ['Cuentos Crueles', 'common.topic.notable_for', 'g.125bv16rg'],\n", + " ['When William Came', 'book.book_edition.binding', 'Paperback'],\n", + " ['Dante Alighieri', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Samuel Beckett', 'common.topic.notable_types', 'Author'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.034vjjt'],\n", + " ['Author',\n", + " 'book.book_subject.works',\n", + " \"Events in 2011 and Forthcoming Book Strengthen James Baldwin's Legacy\"],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01066tpc'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j5g7z0'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010fp24m'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " \"You needn't tell me that a man who doesn't love oysters and asparagus and good wines has got a soul, or a stomach either. He's simply got the instinct for being unhappy highly developed.\"],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8wn'],\n", + " ['Deceased Person', 'type.type.properties', 'Date of death'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Also in 1891, Wilde met Lord Alfred Douglas and became romantically and sexually involved with him.'],\n", + " ['Saki', 'book.author.works_written', 'The novel and plays of Saki'],\n", + " ['Robert W. Chambers', 'people.person.profession', 'Novelist'],\n", + " ['Poet',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Dan Humphrey'],\n", + " ['When William Came', 'book.book_edition.book', 'When William Came'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d521lm'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'Pact with the Devil'],\n", + " ['Humorist', 'common.topic.notable_types', 'Profession'],\n", + " ['John Ruskin',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'oscar wilde taught by john ruskin'],\n", + " ['Publishing',\n", + " 'common.topic.subject_of',\n", + " 'Three publishing trends & their implications'],\n", + " ['Dante Alighieri', 'people.person.profession', 'Poet'],\n", + " ['Dante Alighieri', 'influence.influence_node.influenced', 'James Joyce'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Lucas Scott'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'The Picture of Dorian Gray'],\n", + " ['Paul Merton', 'people.person.nationality', 'United Kingdom'],\n", + " ['Oscar Wilde', 'book.author.book_editions_published', 'Canterville Ghost'],\n", + " ['Writer', 'people.profession.specializations', 'Poet'],\n", + " ['When William Came (Large Print Edition)',\n", + " 'book.book_edition.book',\n", + " 'When William Came'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.011qyfwc'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'The Picture of Dorian Gray'],\n", + " ['The short stories of Saki', 'common.topic.notable_for', 'g.125d7nvkq'],\n", + " ['Mateiu Caragiale',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Edgar Allan Poe'],\n", + " ['Edgar Allan Poe',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Poet'],\n", + " ['The Approaching 100th Anniversary of the Harlem Renaissance (part 2)',\n", + " 'book.written_work.subjects',\n", + " 'Author'],\n", + " ['When William Came', 'book.book_edition.author_editor', 'Saki'],\n", + " ['Christopher Hitchens',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Oscar Wilde'],\n", + " ['When William came', 'common.topic.notable_types', 'Book Edition'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8v7'],\n", + " ['m.034r0v5',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['When William Came', 'common.topic.notable_types', 'Book Edition'],\n", + " ['Authors Frequently Mentioned on the Web',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbss96'],\n", + " ['Writer', 'people.profession.specializations', 'Radio writer'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Morrissey'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0_y5j64'],\n", + " ['When William Came', 'book.book_edition.author_editor', 'Saki'],\n", + " ['Writer', 'people.profession.specializations', 'Speechwriter'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j5g7xv'],\n", + " ['Dublin', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['The sacrifices of friendship were beautiful in her eyes as long as she was not asked to make them.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['When William Came',\n", + " 'book.book.editions',\n", + " 'When William Came (Large Print Edition)'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.01283tsj'],\n", + " ['Dante Alighieri', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Author', 'type.type.properties', 'Book editions published'],\n", + " ['Dorian Gray', 'media_common.netflix_title.netflix_genres', 'Drama'],\n", + " ['Virginia Woolf',\n", + " 'influence.influence_node.influenced_by',\n", + " 'William Shakespeare'],\n", + " ['Cuentos de Humor Negro - C y C -', 'book.written_work.author', 'Saki'],\n", + " ['m.0448y9k',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Logs [Window]', 'film.film.film_art_direction_by', 'Ilva Klavina'],\n", + " ['Honoré de Balzac', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['arthur conan doyle dined with oscar wilde',\n", + " 'base.kwebbase.kwconnection.other',\n", + " 'Oscar Wilde'],\n", + " ['m.07ndk95',\n", + " 'base.ontologies.ontology_instance_mapping.freebase_topic',\n", + " 'Author'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Dupuytren's contracture\"],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'Picture of Dorian Gray (Bookworm Series, Stage 3))'],\n", + " ['Author',\n", + " 'base.ontologies.ontology_instance.equivalent_instances',\n", + " 'm.07ndk95'],\n", + " ['Writer', 'people.profession.specializations', 'Columnist'],\n", + " ['Writer', 'common.topic.article', 'm.0cbdb'],\n", + " ['The cook was a good cook, as cooks go; and as cooks go, she went.',\n", + " 'media_common.quotation.subjects',\n", + " 'Servants'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Actinic keratosis'],\n", + " ['James Abbott McNeill Whistler',\n", + " 'influence.influence_node.influenced',\n", + " 'Oscar Wilde'],\n", + " ['Oscar Wilde', 'people.person.education', 'm.0n0lvt3'],\n", + " ['Walter Pater', 'influence.influence_node.influenced', 'Oscar Wilde'],\n", + " ['Author', 'type.type.properties', 'Series Written (or Contributed To)'],\n", + " ['Logs [Window]', 'film.film.starring', 'm.0pbrr_g'],\n", + " ['Deceased Person', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Deceased Person', 'freebase.type_hints.included_types', 'Person'],\n", + " ['Consumer product', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Saki', 'book.author.works_written', 'The Happy Cat'],\n", + " ['William Shakespeare', 'people.person.nationality', 'England'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'Picture of Dorian Gray (Whole Story)'],\n", + " ['Author', 'type.type.expected_by', 'Author/editor'],\n", + " ['The Importance of Being Earnest',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'United Kingdom'],\n", + " ['Saki', 'book.author.works_written', 'Contador de Historias, El'],\n", + " ['The Unbearable Saki: The Work of H. H. Munro',\n", + " 'book.book.editions',\n", + " 'The Unbearable Saki: The Work of H. H. Munro'],\n", + " ['Surprising Stories by Saki', 'common.topic.notable_for', 'g.125c54_nf'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y9k'],\n", + " ['Lucas Scott', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Author', 'type.type.expected_by', 'Author(s)'],\n", + " ['Oscar Wilde', 'common.topic.article', 'm.05npd'],\n", + " ['Logs [Window]', 'film.film.language', 'Latvian Language'],\n", + " ['Conrad the Wise', 'film.film.other_crew', 'm.0j5g7ll'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville Ghost (Penguin Joint Venture Readers)'],\n", + " ['Plato', 'freebase.valuenotation.has_value', 'Date of death'],\n", + " ['Pancreatic cancer',\n", + " 'medicine.icd_9_cm_classification.parent_classification',\n", + " 'Pancreatic cancer'],\n", + " ['Writer', 'people.profession.specializations', 'Food writer'],\n", + " ['Male',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Attention deficit hyperactivity disorder'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Lemony Snicket'],\n", + " ['Oscar Wilde', 'base.americancomedy.comedian.comedy_genres', 'Word play'],\n", + " ['Oscar Wilde',\n", + " 'influence.influence_node.influenced',\n", + " 'Christopher Hitchens'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Mateiu Caragiale'],\n", + " ['Dante Alighieri', 'people.person.religion', 'Catholicism'],\n", + " ['Playwright', 'base.descriptive_names.names.descriptive_name', 'm.0_z6mdk'],\n", + " ['Writer', 'people.profession.specializations', 'Liner notes author'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Paul Merton'],\n", + " ['Librettist',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Beasts and Super-Beasts', 'common.topic.notable_types', 'Book'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " [\"Jun'ichirō Tanizaki\",\n", + " 'influence.influence_node.influenced',\n", + " 'Fumiko Enchi'],\n", + " ['Camille Paglia', 'influence.influence_node.influenced_by', 'Walter Pater'],\n", + " ['Jorge Luis Borges',\n", + " 'influence.influence_node.influenced_by',\n", + " 'William Shakespeare'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010gbnxw'],\n", + " ['James Abbott McNeill Whistler',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'oscar wilde a pal of james abbott mcneill whistler'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.0_zdby5'],\n", + " ['Morrissey', 'people.person.religion', 'Catholicism'],\n", + " ['m.0pbrrh0', 'film.performance.film', 'Logs [Window]'],\n", + " ['Consumer product', 'freebase.type_profile.published', 'Published'],\n", + " ['Edgar Allan Poe', 'influence.influence_node.influenced', 'Oscar Wilde'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1v3z'],\n", + " ['Robert W. Chambers',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Edgar Allan Poe'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gastritis'],\n", + " ['English Language',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'Republic of Ireland'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Sebaceous cyst'],\n", + " ['Aristotle', 'people.person.profession', 'Writer'],\n", + " ['James Joyce', 'influence.influence_node.influenced_by', 'Aristotle'],\n", + " ['Sittwe', 'common.topic.image', 'Shwe zedi'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'James K. Morrow'],\n", + " ['William Godwin', 'people.person.profession', 'Novelist'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'camille pissarro met oscar wilde'],\n", + " ['No one has ever said it, but how painfully true it is that the poor have us always with them.',\n", + " 'media_common.quotation.subjects',\n", + " 'Riches'],\n", + " ['Oscar Wilde', 'film.film_story_contributor.film_story_credits', 'Salome'],\n", + " ['Logs [Window]', 'film.film.film_set_decoration_by', 'Nauris Stalazs'],\n", + " ['The Chronicles of Clovis', 'common.topic.article', 'm.05985n3'],\n", + " ['Brandon M. Dennis', 'people.person.gender', 'Male'],\n", + " ['When William Came',\n", + " 'book.written_work.next_in_series',\n", + " 'Beasts and Super-Beasts'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0ws2n8_'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbstjs'],\n", + " ['The Fan', 'film.film.story_by', 'Oscar Wilde'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbstlv'],\n", + " ['Honoré de Balzac',\n", + " 'influence.influence_node.influenced',\n", + " 'Mateiu Caragiale'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbsslb'],\n", + " ['Mary Frances Mercer', 'people.person.spouse_s', 'm.0nf30nj'],\n", + " ['Wit',\n", + " 'base.americancomedy.comedy_genre.comedians_in_this_genre',\n", + " 'William Shakespeare'],\n", + " ['Playwright', 'base.descriptive_names.names.descriptive_name', 'm.0_z6mct'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010fp1_3'],\n", + " ['United Kingdom',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'England'],\n", + " ['m.0j5g7x7', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Bard', 'common.topic.notable_types', 'Profession'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'walter sickert stayed with oscar wilde'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.04p7zsw'],\n", + " ['The Toys of Peace, and Other Papers',\n", + " 'book.book.editions',\n", + " 'The toys of peace, and other papers'],\n", + " ['England',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United Kingdom'],\n", + " ['Person', 'type.type.properties', 'Gender'],\n", + " ['Charles Augustus Munro', 'people.person.spouse_s', 'm.0nf30nj'],\n", + " ['m.0nf33dh', 'film.performance.film', 'Sredni Vashtar'],\n", + " ['Dave Eggers', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Satirist', 'common.topic.notable_types', 'Profession'],\n", + " ['When William Came',\n", + " 'book.book.editions',\n", + " 'When William came; a story of London under the Hohenzollerns'],\n", + " ['Consumer product', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Amanda Filipacchi', 'people.person.profession', 'Novelist'],\n", + " ['Edgar Allan Poe', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Sredni Vashtar', 'film.film.cinematography', 'Peter Hannan'],\n", + " ['André Gide', 'people.person.place_of_birth', 'Paris'],\n", + " ['Author',\n", + " 'book.book_subject.works',\n", + " 'The Approaching 100th Anniversary of the Harlem Renaissance (part 2)'],\n", + " ['Oscar Wilde', 'people.person.spouse_s', 'm.0hygn49'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0nflmmh'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02_98hd'],\n", + " ['Irène Némirovsky', 'people.person.gender', 'Female'],\n", + " ['André Gide', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['Female', 'common.topic.subjects', 'ZasPorn'],\n", + " ['Edgar Allan Poe', 'influence.influence_node.influenced', 'Walt Whitman'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Wilde also associated with the fin de siecle decadent and symbolist poets and writers of the period living in France and England.'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8vz'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['Author',\n", + " 'common.topic.subject_of',\n", + " 'The Passion-Driven Writer and the Digital-Age Literary Marketplace'],\n", + " ['Victor Hugo',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'oscar wilde met victor hugo'],\n", + " ['m.0j5g7ll', 'film.film_crew_gig.film', 'Conrad the Wise'],\n", + " ['When William Came (Large Print)',\n", + " 'book.book_edition.binding',\n", + " 'Paperback'],\n", + " ['Jorge Luis Borges', 'people.person.profession', 'Essayist'],\n", + " ['m.0r5qxs_', 'people.sibling_relationship.sibling', 'Oscar Wilde'],\n", + " ['Honoré de Balzac', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['Aldous Huxley', 'people.person.nationality', 'United Kingdom'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Picture of Dorian Gray'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['The background', 'common.topic.notable_for', 'g.125bwrkzd'],\n", + " ['Amanda Filipacchi', 'people.person.religion', 'Atheism'],\n", + " ['Stephen Fry', 'people.person.profession', 'Novelist'],\n", + " ['Walt Whitman',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'walt whitman a fan was oscar wilde'],\n", + " ['m.0nfq0_4',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Publishing',\n", + " 'book.book_subject.works',\n", + " 'The Passion-Driven Writer and the Digital-Age Literary Marketplace'],\n", + " ['Giving',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " 'When people grow gradually rich their requirements and standard of living expand in proportion, while their present-giving instincts often remain in the undeveloped condition of their earlier days. Something showy and not-too-expensive in a shop is their only conception of the ideal gift.'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " \"Lord Arthur Savile's Crime\"],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d5218t'],\n", + " ['Novelist',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'David Wilson'],\n", + " ['Clasicos De Suspenso/ Classics Of Supense',\n", + " 'common.topic.notable_for',\n", + " 'g.1255m89wz'],\n", + " ['m.0pbss96', 'film.film_crew_gig.film', 'Logs [Window]'],\n", + " ['Writer', 'people.profession.part_of_professional_field', 'Publishing'],\n", + " ['The Unbearable Saki: The Work of H. H. Munro',\n", + " 'book.written_work.subjects',\n", + " 'England'],\n", + " ['A Good Woman', 'common.topic.notable_types', 'Film'],\n", + " ['We all know that Prime Ministers are wedded to the truth, but like other wedded couples they sometimes live apart.',\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['The Playboy of the Weekend World', 'common.topic.notable_types', 'Book'],\n", + " [\"Hors d'oeuvres have always a pathetic interest for me; they remind me of one's childhood that one goes through wondering what the next course is going to be like -- and during the rest of the menu one wishes one had eaten more of the hors d'oeuvres.\",\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['Oscar Wilde',\n", + " 'visual_art.art_subject.artwork_on_the_subject',\n", + " 'A Private View at the Royal Academy, 1881'],\n", + " ['Authors Frequently Mentioned on the Web',\n", + " 'book.written_work.subjects',\n", + " 'Literature'],\n", + " ['Amanda Filipacchi',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Oscar Wilde'],\n", + " ['Literature',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '\\\\\"It’s a good thing that he [John Galsworthy] and those who answered his call in the twentieth century have given the rest of us tilling fields in the twenty-first century a 90-year head start.\\\\\"'],\n", + " ['m.0gxvv0q',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Max Beerbohm', 'people.person.profession', 'Novelist'],\n", + " ['m.0j5g7m4', 'film.film_crew_gig.crewmember', 'Adam Fulton'],\n", + " ['Sredni Vashtar', 'common.topic.image', 'm.0hp0cfz'],\n", + " ['m.04_m4gz', 'media_common.dedication.dedicated_by', 'Oscar Wilde'],\n", + " ['The Importance of Being Earnest', 'common.topic.notable_types', 'Film'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Corneal abrasion'],\n", + " ['James Joyce', 'people.person.nationality', 'Republic of Ireland'],\n", + " ['When William came', 'book.book_edition.author_editor', 'Saki'],\n", + " ['Sittwe', 'location.location.contains', 'Dhanyawaddy Stadium'],\n", + " ['Dorian Gray', 'film.film.country', 'United Kingdom'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64q8'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " 'Poverty keeps together more homes than it breaks up.'],\n", + " ['m.02zd4tr',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Oscar Wilde', 'common.topic.notable_for', 'g.1258mvrbj'],\n", + " ['Author', 'base.ontologies.ontology_class.equivalent_classes', 'm.0dj2h10'],\n", + " ['Playwright', 'education.field_of_study.students_majoring', 'm.012nj6qx'],\n", + " ['James K. Morrow',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Dante Alighieri'],\n", + " ['Julius Evola', 'kp_lw.philosopher.influenced_by', 'Dante Alighieri'],\n", + " ['Scandal is merely the compassionate allowance which the gay make to the humdrum. Think how many blameless lives are brightened by the blazing indiscretions of other people.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Conrad the Wise', 'film.film.release_date_s', 'm.0j5g7b6'],\n", + " ['Sittwe', 'common.topic.image', 'MyanmarRakhine'],\n", + " ['The Unbearable Bassington', 'common.topic.notable_for', 'g.125ck1vc6'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8t9'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010hvjyd'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney cancer'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0_y5j47'],\n", + " ['W. H. Pugmire', 'people.person.profession', 'Writer'],\n", + " ['Cuentos Crueles', 'common.topic.notable_types', 'Book'],\n", + " ['Peter Kropotkin',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'peter alexeyevich kropotkin a supporter was oscar wilde'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " \"The trial, which began well for Wilde, but soon deteriorated when it became clear that Queensberry's lawyers had gathered ample evidence of Wilde's homosexual affairs.\"],\n", + " ['Walt Whitman', 'people.person.profession', 'Writer'],\n", + " ['Irène Némirovsky',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Oscar Wilde'],\n", + " ['James K. Morrow', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville ghost'],\n", + " ['We all know that Prime Ministers are wedded to the truth, but like other wedded couples they sometimes live apart.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Virginia Woolf', 'people.person.gender', 'Female'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'Picture of Dorian Gray (Cassette) (Cdl 51095)'],\n", + " ['Jorge Luis Borges',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Oscar Wilde'],\n", + " ['Julius Evola', 'people.person.gender', 'Male'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Picture of Dorian Gray (AUDIOBOOK) (AUDIO CD)'],\n", + " ['Aldous Huxley', 'people.person.profession', 'Author'],\n", + " ['The Passion-Driven Writer and the Digital-Age Literary Marketplace',\n", + " 'book.written_work.subjects',\n", + " 'Author'],\n", + " ['Novelist', 'common.topic.notable_types', 'Profession'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8qb'],\n", + " ['Sittwe', 'location.location.people_born_here', 'U Ottama'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Julius Evola', 'people.person.profession', 'Writer'],\n", + " ['Publishing',\n", + " 'people.professional_field.professions_in_this_field',\n", + " 'Writer'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Kitty Walker'],\n", + " ['Logs [Window]', 'film.film.film_set_decoration_by', 'Janis Straupe'],\n", + " [\"It's no use growing older if you only learn new ways of misbehaving yourself.\",\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Writer', 'people.profession.specializations', 'Bard'],\n", + " ['Oscar Wilde', 'people.person.nationality', 'Republic of Ireland'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y9y'],\n", + " ['Poverty keeps together more homes than it breaks up.',\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['The Chronicles of Clovis', 'book.written_work.author', 'Saki'],\n", + " ['Author', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Beaumont-Hamel',\n", + " 'common.topic.image',\n", + " 'Beaumont hamel newfoundland memorial'],\n", + " ['Conrad the Wise', 'common.topic.webpage', 'm.0j51ccz'],\n", + " ['Dorian Gray', 'film.film.genre', 'Thriller'],\n", + " [\"Great Socialist statesmen aren't made, they're still-born.\",\n", + " 'media_common.quotation.subjects',\n", + " 'Socializing and Socialism'],\n", + " ['Oscar Wilde', 'base.popstra.celebrity.dated', 'm.0gyj7hl'],\n", + " ['m.034vjjt',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Edgar Allan Poe', 'people.person.profession', 'Novelist'],\n", + " ['Authors Frequently Mentioned on the Web',\n", + " 'book.written_work.subjects',\n", + " 'James Joyce'],\n", + " ['heart attack', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Writer',\n", + " 'book.book_subject.works',\n", + " 'The Passion-Driven Writer and the Digital-Age Literary Marketplace'],\n", + " ['André Gide', 'influence.influence_node.influenced_by', 'Victor Hugo'],\n", + " ['When William Came', 'book.book.editions', 'When William Came'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Kilgore Trout'],\n", + " ['Sredni Vashtar', 'common.topic.notable_for', 'g.125d5v3ds'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['When William Came', 'common.topic.notable_for', 'g.1258jn545'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Urolithiasis'],\n", + " ['Phone Trick',\n", + " 'base.thoroughbredracing.thoroughbred_racehorse.sex',\n", + " 'Male'],\n", + " ['Dante Alighieri', 'influence.influence_node.influenced', 'Julius Evola'],\n", + " ['Incredible tales', 'book.written_work.author', 'Saki'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Max Beerbohm'],\n", + " ['Emma Goldman', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['William Shakespeare',\n", + " 'fictional_universe.fictional_character.based_on',\n", + " 'William Shakespeare'],\n", + " ['Playwright', 'common.topic.article', 'm.09sfv'],\n", + " ['Edgar Allan Poe',\n", + " 'influence.influence_node.influenced',\n", + " 'Jorge Luis Borges'],\n", + " ['Beasts and Super-Beasts', 'book.book.editions', 'Beasts & superbeasts'],\n", + " ['Beaumont-Hamel', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.034r0v5'],\n", + " ['Sittwe', 'location.location.containedby', 'Sittwe District'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0cjm4sb'],\n", + " ['m.0pbrrvb', 'film.performance.film', 'Logs [Window]'],\n", + " ['Saki', 'book.author.works_written', 'Sredni Vashtar'],\n", + " ['Paris', 'base.biblioness.bibs_topic.subsumes', 'Paris'],\n", + " ['Honoré de Balzac', 'people.person.profession', 'Novelist'],\n", + " ['The Unbearable Bassington',\n", + " 'book.written_work.next_in_series',\n", + " 'When William Came'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " \"He's simply got the instinct for being unhappy highly developed.\"],\n", + " ['Writer', 'common.topic.notable_for', 'g.1258kfqgd'],\n", + " ['Matt Moniz', 'common.topic.subjects', 'Writer'],\n", + " ['Novelist', 'common.topic.subject_of', 'Kiersten Fay'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Wilde is recognized today as one of the great forces of British literature from the turn of the century.'],\n", + " ['Writer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Person', 'freebase.type_profile.strict_included_types', 'Agent'],\n", + " ['Sredni Vashtar',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.08dd6hh'],\n", + " ['m.0j5g7lx', 'film.film_crew_gig.film', 'Conrad the Wise'],\n", + " ['Walt Whitman', 'people.person.profession', 'Journalist'],\n", + " ['He spends his life explaining from his pulpit that the glory of Christianity consists in the fact that though it is not true it has been found necessary to invent it.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Oscar Wilde', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['The Passion-Driven Writer and the Digital-Age Literary Marketplace',\n", + " 'common.topic.subjects',\n", + " 'Author'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Ernest Pratt'],\n", + " ['Samuel Beckett', 'people.deceased_person.place_of_death', 'Paris'],\n", + " ['Peter Kropotkin', 'people.person.profession', 'Writer'],\n", + " ['Oscar Wilde',\n", + " 'influence.influence_node.influenced_by',\n", + " 'James Abbott McNeill Whistler'],\n", + " ['The Unbearable Bassington',\n", + " 'book.book.editions',\n", + " 'The Unbearable Bassington'],\n", + " ['Saki', 'book.author.works_written', 'The miracle-merchant'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'André Gide'],\n", + " ['Dave Sim', 'people.person.profession', 'Novelist'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.includes_causes_of_death',\n", + " 'Myocardial Ischemia'],\n", + " ['Oscar Wilde',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Christopher Hitchens', 'people.person.profession', 'Essayist'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Hair loss'],\n", + " [\"L'insupportable Bassington\", 'common.topic.notable_for', 'g.1257jm4hq'],\n", + " ['m.0bnmr38', 'base.prison.imprisonment.prisoner', 'Oscar Wilde'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.0gxvv0q'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1v59'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y91'],\n", + " ['Beasts and Super-Beasts', 'book.book.editions', 'Beasts and super-beasts'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq91h'],\n", + " ['The Square Egg',\n", + " 'book.book.editions',\n", + " 'The square egg and other sketches, with three plays'],\n", + " ['m.0kfyrv7',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.0_zdbyn'],\n", + " ['World War I',\n", + " 'base.culturalevent.event.entity_involved',\n", + " 'British Empire'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j5g7wt'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Syphilis'],\n", + " ['Oscar Wilde', 'people.person.ethnicity', 'Irish people'],\n", + " ['Sredni Vashtar', 'film.film.runtime', 'm.0nfh0mz'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010gbnzx'],\n", + " ['Logs [Window]',\n", + " 'film.film.film_festivals',\n", + " '2012 Lielais Kristaps National Film Festival'],\n", + " ['English Language',\n", + " 'language.human_language.main_country',\n", + " 'Republic of Ireland'],\n", + " [\"Parkinson's disease\", 'medicine.disease.risk_factors', 'Male'],\n", + " ['The rise of the Russian empire', 'book.written_work.author', 'Saki'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Stephen Fry'],\n", + " ['Writer', 'common.topic.subject_of', 'Matt Moniz'],\n", + " ['Wilde', 'film.film.country', 'United Kingdom'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Hirschsprung's disease\"],\n", + " ['Picardy',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Somme'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'He wrote sonnets to Ellen Terry.'],\n", + " ['Logs [Window]', 'common.topic.notable_types', 'Film'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1w0z'],\n", + " ['Honoré de Balzac', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Stroke', 'medicine.disease.risk_factors', 'Transient ischemic attack'],\n", + " ['Mateiu Caragiale', 'common.topic.notable_types', 'Author'],\n", + " ['The cook was a good cook, as cooks go; and as cooks go, she went.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'An Ideal Husband'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Multiple myeloma'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010fp299'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Picture of Dorian Gray'],\n", + " ['Saki', 'book.author.works_written', 'Beasts and Super-Beasts'],\n", + " ['When William Came', 'book.book.genre', 'Alternate history'],\n", + " ['Oscar Wilde', 'book.author.book_editions_published', 'Canterville Ghost'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'The novel generated many negative reviews from readers who were disturbed by the veiled references to homosexuality.'],\n", + " ['Oscar Wilde', 'base.popstra.celebrity.eats_at', 'm.0gyj79z'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'oscar wilde taught by john ruskin'],\n", + " ['William Godwin', 'people.person.profession', 'Journalist'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Around this time Wilde also published the pamphlet, The Soul of Man Under Socialism, which reflected his growing interest in libertarian socialism, as well as the influence of anarchist Kropotkin.'],\n", + " ['m.0nf2pdl',\n", + " 'film.film_regional_release_date.film_release_region',\n", + " 'United Kingdom'],\n", + " ['Jorge Luis Borges', 'people.person.profession', 'Author'],\n", + " ['m.0j5g7b6', 'film.film_regional_release_date.film', 'Conrad the Wise'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Myocardial Ischemia'],\n", + " ['Poet', 'people.profession.specialization_of', 'Writer'],\n", + " ['Author', 'type.type.properties', 'Works edited'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Carrie Bradshaw'],\n", + " ['m.0pbrs7f', 'film.performance.film', 'Logs [Window]'],\n", + " ['John Keats', 'people.person.gender', 'Male'],\n", + " ['Dorian Gray', 'film.film.genre', 'Drama'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'He left England and lived in France, where he wrote his last great work, The Ballad of Reading Gaol, a long poem that was about both his prison experience and his relationship with Alfred Douglas.'],\n", + " ['Word play',\n", + " 'base.americancomedy.comedy_genre.comedians_in_this_genre',\n", + " 'Oscar Wilde'],\n", + " ['Oscar Wilde', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['When William Came',\n", + " 'book.written_work.previous_in_series',\n", + " 'The Unbearable Bassington'],\n", + " ['Oscar Wilde', 'people.person.profession', 'Journalist'],\n", + " ['m.0j50nj6', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['The Un-Rest Cure', 'common.topic.notable_types', 'Book'],\n", + " ['m.0nf30x3', 'people.sibling_relationship.sibling', 'Ethel Munro'],\n", + " ['When people grow gradually rich their requirements and standard of living expand in proportion, while their present-giving instincts often remain in the undeveloped condition of their earlier days. Something showy and not-too-expensive in a shop is their only conception of the ideal gift.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Christopher Hitchens',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date of death'],\n", + " ['Author',\n", + " 'tv.tv_subject.tv_episodes',\n", + " 'J. J. Woods – Author - The Forgotten Covenant'],\n", + " ['m.0gyj79z', 'base.popstra.restaurant_choice.diner', 'Oscar Wilde'],\n", + " ['Samuel Beckett', 'people.person.profession', 'Novelist'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Samuel Beckett', 'people.person.profession', 'Author'],\n", + " ['Oscar Wilde', 'influence.influence_node.peers', 'm.0gym3p5'],\n", + " ['James Joyce',\n", + " 'book.book_subject.works',\n", + " 'Authors Frequently Mentioned on the Web'],\n", + " ['Stephen Fry',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Christopher Hitchens'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0zm9_69'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'Dorian Gray'],\n", + " ['Screenwriter', 'people.profession.specialization_of', 'Writer'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pancreatic cancer'],\n", + " ['Victor Hugo', 'people.person.profession', 'Playwright'],\n", + " ['Author', 'type.type.expected_by', 'Contributing authors'],\n", + " ['Writer', 'people.profession.specializations', 'Website content writer'],\n", + " ['Saki', 'book.author.works_written', 'Cuentos Crueles'],\n", + " ['Oscar Wilde', 'people.deceased_person.cause_of_death', 'Meningitis'],\n", + " ['Honoré de Balzac', 'common.topic.notable_types', 'Author'],\n", + " ['Oscar Wilde', 'common.topic.webpage', 'm.0b6s0jc'],\n", + " ['m.04p7zsw',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010bq8xl'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvjy3'],\n", + " ['Dublin', 'location.location.containedby', 'Republic of Ireland'],\n", + " ['Author', 'type.type.expected_by', 'Editor'],\n", + " ['Edgar Allan Poe portrait B',\n", + " 'common.image.appears_in_topic_gallery',\n", + " 'Edgar Allan Poe'],\n", + " ['The Importance of Being Earnest', 'film.film.genre', 'Drama'],\n", + " ['Author', 'type.type.expected_by', 'Editor'],\n", + " ['Englische Kurzgesschichten', 'common.topic.notable_types', 'Book'],\n", + " ['Author', 'type.type.properties', 'Open Library ID'],\n", + " ['m.0pbssf2', 'film.film_crew_gig.film', 'Logs [Window]'],\n", + " ['Edgar Allan Poe', 'people.person.profession', 'Author'],\n", + " ['Victor Hugo',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Saki',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'Conrad the Wise'],\n", + " ['Author', 'type.type.expected_by', 'Author/editor'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0bm6pqb'],\n", + " ['Beaumont-Hamel', 'location.location.containedby', 'France'],\n", + " ['Saki', 'book.author.works_written', 'The Playboy of the Weekend World'],\n", + " ['Oscar Wilde', 'common.topic.webpage', 'm.09y2zb5'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Benjamin Tucker'],\n", + " ['Christopher Hitchens', 'people.person.profession', 'Journalist'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'The Selfish Giant'],\n", + " ['Deceased Person', 'freebase.type_profile.published', 'Published'],\n", + " ['Playwright', 'common.topic.notable_types', 'Profession'],\n", + " ['Publishing',\n", + " 'book.book_subject.works',\n", + " 'Tough Love from Author Enablers Helps Get the Writing Job Done'],\n", + " ['Cuentos Indiscretos', 'book.written_work.author', 'Saki'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64rh'],\n", + " ['Essayist', 'common.topic.notable_types', 'Profession'],\n", + " ['Stephen Fry', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['Dante Alighieri', 'influence.influence_node.influenced', 'Samuel Beckett'],\n", + " ['Victor Hugo', 'freebase.valuenotation.is_reviewed', 'Date of death'],\n", + " ['Irène Némirovsky', 'people.person.religion', 'Catholicism'],\n", + " ['m.0n5lsrr', 'award.award_nomination.award_nominee', 'Oscar Wilde'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Marcie Walsh McBain'],\n", + " ['Poet',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Writer', 'people.profession.specializations', 'Feminist writer'],\n", + " ['Author', 'type.type.expected_by', 'Editor'],\n", + " ['MA$TADON', 'people.person.profession', 'Screenwriter'],\n", + " ['Oscar Wilde', 'people.person.children', 'Vyvyan Holland'],\n", + " ['Publishing', 'book.book_subject.works', 'Write That Book Already!'],\n", + " ['Sittwe', 'location.location.containedby', 'Sittwe Township'],\n", + " ['Writer', 'common.topic.notable_types', 'Profession'],\n", + " ['William Shakespeare', 'base.americancomedy.comedian.comedy_genres', 'Wit'],\n", + " ['Science writer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Esophageal cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Fiction', 'media_common.media_genre.child_genres', 'Mystery'],\n", + " ['James Joyce', 'common.topic.notable_types', 'Author'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Esophageal cancer'],\n", + " ['Samuel Beckett', 'people.person.profession', 'Writer'],\n", + " ['m.0j5g7w6', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Author',\n", + " 'book.book_subject.works',\n", + " 'Tough Love from Author Enablers Helps Get the Writing Job Done'],\n", + " ['Screenwriter',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Sredni Vashtar', 'book.written_work.author', 'Saki'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Edmund Grey'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Robert W. Chambers'],\n", + " ['Poverty keeps together more homes than it breaks up.',\n", + " 'media_common.quotation.subjects',\n", + " 'Poverty and The Poor'],\n", + " ['Society of Authors',\n", + " 'organization.organization.geographic_scope',\n", + " 'United Kingdom'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0bj8dwd'],\n", + " ['Oscar Wilde',\n", + " 'people.deceased_person.place_of_burial',\n", + " 'Cimetière parisien de Bagneux'],\n", + " ['Fumiko Enchi', 'people.deceased_person.cause_of_death', 'heart attack'],\n", + " ['Julius Evola', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['m.0nfgyzh', 'film.film_crew_gig.film', 'Sredni Vashtar'],\n", + " ['Sittwe', 'common.topic.article', 'm.05ql95'],\n", + " ['m.0j5g7zp', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Deceased Person', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['When William Came', 'book.written_work.author', 'Saki'],\n", + " ['When William Came', 'book.book_edition.isbn', '9781598184822'],\n", + " ['m.0448xjq',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['The Toys of Peace, and Other Papers', 'book.written_work.author', 'Saki'],\n", + " ['Logs [Window]', 'film.film.starring', 'm.0pbrrvb'],\n", + " ['Oscar Wilde', 'freebase.valuenotation.is_reviewed', 'Date of death'],\n", + " ['Writer', 'projects.project_role.projects', 'm.0y50gnx'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Tom Stoppard'],\n", + " ['Dave Sim', 'people.person.gender', 'Male'],\n", + " ['Conrad the Wise',\n", + " 'film.film.film_festivals',\n", + " '2009 Raindance Film Festival'],\n", + " ['Logs [Window]', 'film.film.genre', 'Short Film'],\n", + " ['James K. Morrow', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['He is one of those people who would be enormously improved by death.',\n", + " 'media_common.quotation.subjects',\n", + " 'People'],\n", + " ['When William Came', 'book.book.editions', 'When William Came'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j5g7y8'],\n", + " ['Saki', 'people.person.nationality', 'United Kingdom'],\n", + " ['John Ruskin', 'people.person.profession', 'Poet'],\n", + " ['Christopher Hitchens', 'people.person.nationality', 'United Kingdom'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'walt whitman a fan was oscar wilde'],\n", + " ['Oscar Wilde', 'base.prison.prisoner.imprisoned', 'm.0bnmr38'],\n", + " ['The Complete Saki', 'book.written_work.author', 'Saki'],\n", + " ['The Complete Saki', 'book.book.genre', 'Speculative fiction'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Amyotrophic lateral sclerosis'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.011nr3wd'],\n", + " ['Author', 'type.type.expected_by', 'Author/editor'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'A Woman of No Importance'],\n", + " ['Award-Winning Work', 'freebase.type_profile.published', 'Published'],\n", + " ['Saki', 'book.author.works_written', 'The Chronicles of Clovis'],\n", + " ['Writer', 'people.profession.specializations', 'Science-fiction author'],\n", + " ['m.0nf332r', 'film.performance.film', 'Sredni Vashtar'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y9f'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j50nj6'],\n", + " ['Walt Whitman',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Edgar Allan Poe'],\n", + " ['Jorge Luis Borges', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Male',\n", + " 'base.skosbase.vocabulary_equivalent_topic.equivalent_concept',\n", + " 'Males'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbss_n'],\n", + " ['m.05bvk3z',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['m.0ztr7jq', 'award.award_honor.award_winner', 'Oscar Wilde'],\n", + " ['Edgar Allan Poe', 'influence.influence_node.influenced', 'Fumiko Enchi'],\n", + " ['Sittwe', 'location.location.people_born_here', 'Aung Lwin'],\n", + " ['Robert W. Chambers', 'common.topic.notable_types', 'Author'],\n", + " ['Author', 'freebase.documented_object.documentation', 'm.021y5yr'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j5g7x7'],\n", + " ['Walt Whitman', 'people.person.profession', 'Poet'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['When William Came', 'book.book.genre', 'Fiction'],\n", + " ['Writer', 'people.profession.specializations', 'Political writer'],\n", + " ['Emma Goldman', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Ethel Munro', 'common.topic.notable_types', 'Person'],\n", + " [\"You needn't tell me that a man who doesn't love oysters and asparagus and good wines has got a soul, or a stomach either. He's simply got the instinct for being unhappy highly developed.\",\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Saki', 'people.person.parents', 'Charles Augustus Munro'],\n", + " ['Author', 'type.type.expected_by', 'Editor'],\n", + " ['Beasts and Super-Beasts', 'book.book.genre', 'Speculative fiction'],\n", + " ['m.0g9p00h',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['m.0hp0cfz', 'common.image.appears_in_topic_gallery', 'Sredni Vashtar'],\n", + " ['Writer',\n", + " 'people.profession.represented_by_trade_unions',\n", + " 'Society of Authors'],\n", + " ['Walter Pater', 'people.person.gender', 'Male'],\n", + " ['The Unbearable Saki: The Work of H. H. Munro',\n", + " 'book.book.editions',\n", + " 'The unbearable Saki : the work of H.H. Munro'],\n", + " ['Conrad the Wise', 'film.film.edited_by', 'Alan Miller'],\n", + " ['Kiersten Fay', 'people.person.profession', 'Author'],\n", + " ['m.0bj8dwd',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Writer', 'people.profession.specializations', 'Writer & Actor'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Wilde continued to travel, and after returning to London in early 1883, he quickly left again for Paris.'],\n", + " ['Walter Pater', 'people.person.nationality', 'United Kingdom'],\n", + " ['m.0b6s0jc', 'common.webpage.topic', 'Oscar Wilde'],\n", + " ['Satirist', 'people.profession.specialization_of', 'Writer'],\n", + " ['A Saki sampler', 'common.topic.notable_types', 'Book'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448yfx'],\n", + " ['Sredni Vashtar', 'film.film.directed_by', 'Andrew Birkin'],\n", + " ['Mateiu Caragiale', 'people.person.profession', 'Journalist'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced_by', 'William Godwin'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010l0dn2'],\n", + " ['W. H. Pugmire', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced_by', 'John Keats'],\n", + " ['Female', 'medicine.risk_factor.diseases', 'Kyphosis'],\n", + " ['Jorge Luis Borges',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Edgar Allan Poe'],\n", + " ['When William Came (Large Print)',\n", + " 'book.book_edition.isbn',\n", + " '9781846371578'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " \"All of Wilde's books were taken out of print, his name expunged from posters and announcements for all his plays, and his wife and children had to go into hiding.\"],\n", + " ['Sredni Vashtar, and other stories', 'book.written_work.author', 'Saki'],\n", + " ['“Sociologically, politically, psychologically, spiritually, it was never enough for him [James Baldwin] to categorize himself as one thing or the other: not just black, not just sexual, not just American, nor even just as a world-class literary artist. He embraced the whole of life the way the sun’s gravitational passion embraces everything from the smallest wandering comet to the largest looming planet. He both confronted and cultivated creative vision with a drive, passion, and brilliance that few have matched, and simply being able to watch his genius sparkle from one sentence to the next could generate both awe and revelation.”',\n", + " 'media_common.quotation.subjects',\n", + " 'Writer'],\n", + " ['André Gide', 'people.person.nationality', 'France'],\n", + " ['John Keats', 'influence.influence_node.influenced', 'Jorge Luis Borges'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " \"Wilde's trial began on 26 April, but when the jury could not reach a verdict a second trial was ordered.\"],\n", + " ['Saki',\n", + " 'book.author.book_editions_published',\n", + " 'When William Came (Large Print)'],\n", + " ['m.0j5g7z0', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Picardy',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'France'],\n", + " ['The short stories of Saki',\n", + " 'book.book.editions',\n", + " 'The complete works of Saki'],\n", + " ['Reginald in Russia and Other Sketches',\n", + " 'common.topic.notable_for',\n", + " 'g.1256cph0q'],\n", + " ['Sredni Vashtar', 'common.topic.notable_for', 'g.125bk5y7y'],\n", + " ['SFX Award for Best Actor',\n", + " 'award.award_category.disciplines_or_subjects',\n", + " 'Male'],\n", + " ['The Westminster Alice', 'book.written_work.author', 'Saki'],\n", + " ['Conrad the Wise', 'film.film.starring', 'm.0j50nhx'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0nfq0_4'],\n", + " ['Surprising Stories by Saki', 'book.written_work.author', 'Saki'],\n", + " ['Saki', 'people.person.profession', 'Playwright'],\n", + " ['Logs [Window]', 'film.film.directed_by', 'Una Rozenbauma'],\n", + " ['Kiersten Fay', 'common.topic.subjects', 'Author'],\n", + " ['Selected short stories of Saki (H.H. Munro)',\n", + " 'book.book.editions',\n", + " 'The short stories of Saki (H. H. Munro)'],\n", + " ['Author', 'type.type.properties', 'School or Movement'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " 'No one can be an unbeliever nowadays. The Christian Apologists have left one nothing to disbelieve.'],\n", + " ['The Unbearable Bassington', 'book.book.genre', 'Speculative fiction'],\n", + " ['Clasicos De Suspenso/ Classics Of Supense',\n", + " 'book.written_work.author',\n", + " 'Saki'],\n", + " ['When William Came', 'book.book.editions', 'When William Came'],\n", + " ['William Shakespeare', 'people.person.profession', 'Author'],\n", + " ['English Language',\n", + " 'language.human_language.main_country',\n", + " 'United Kingdom'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04mmfr8'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'In 1880 Wilde and his friend, artist Frank Miles, lived across the street from Whistler.'],\n", + " ['MA$TADON', 'music.composer.compositions', '\\\\\"Strickly O.G\\' Remind-U'],\n", + " ['Olivia Wilde', 'symbols.namesake.named_after', 'Oscar Wilde'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Walt Whitman'],\n", + " ['Comic Book Creator', 'people.profession.specialization_of', 'Author'],\n", + " ['Author', 'type.type.expected_by', 'Author'],\n", + " ['Oscar Wilde', 'common.topic.webpage', 'm.0cq1qlg'],\n", + " ['No one can be an unbeliever nowadays. The Christian Apologists have left one nothing to disbelieve.',\n", + " 'media_common.quotation.subjects',\n", + " 'Atheism'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010fm11p'],\n", + " ['When William Came', 'common.topic.notable_for', 'g.1255p7c9p'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.0kfyrt3'],\n", + " ['William Shakespeare',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Playwright'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'oscar wilde met victor hugo'],\n", + " [\"Events in 2011 and Forthcoming Book Strengthen James Baldwin's Legacy\",\n", + " 'book.written_work.subjects',\n", + " 'Literature'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " \"Wilde later retaliated by publishing a negative review of Whistler's book, The Gentle Art of Making Enemies (1890).\"],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.0106ctrt'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Happy Prince and other Stories. CD. Originaltexte mit Wortschatzhilfen im Begleitheft. (Lernmaterialien)'],\n", + " ['Reginald', 'book.book.editions', 'Reginald'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'johnston forbes-robertson a pal of oscar wilde'],\n", + " ['John Ruskin', 'common.topic.notable_types', 'Author'],\n", + " ['Oscar Wilde',\n", + " 'influence.influence_node.influenced',\n", + " \"Jun'ichirō Tanizaki\"],\n", + " ['Author', 'common.topic.subjects', 'Jeremy Yablan'],\n", + " ['England', 'location.administrative_division.country', 'United Kingdom'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.0106cxtw'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Cardiovascular disease'],\n", + " ['m.0m_2scq', 'people.sibling_relationship.sibling', 'Oscar Wilde'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pyloric stenosis'],\n", + " ['Oscar Wilde', 'common.topic.image', 'Oscar Wilde 3g07095u'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " \"It's no use growing older if you only learn new ways of misbehaving yourself.\"],\n", + " ['Olivia Wilde', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Book', 'freebase.type_profile.published', 'Published'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Auguste Gusteau'],\n", + " ['England',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United Kingdom'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Irène Némirovsky'],\n", + " ['Kiersten Fay', 'common.topic.subjects', 'Writer'],\n", + " ['Englische Kurzgesschichten', 'book.written_work.author', 'Aldous Huxley'],\n", + " ['Saki', 'book.author.works_written', 'The Bodley Head Saki'],\n", + " ['Tobermory and Other Stories', 'common.topic.notable_for', 'g.125gbh5rh'],\n", + " ['Wilde', 'film.film.subjects', 'Oscar Wilde'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvkm6'],\n", + " ['Saki', 'book.author.book_editions_published', 'When William Came'],\n", + " ['Ethel Munro', 'people.person.gender', 'Female'],\n", + " ['Kiersten Fay', 'common.topic.subjects', 'Novelist'],\n", + " ['Scandal is merely the compassionate allowance which the gay make to the humdrum. Think how many blameless lives are brightened by the blazing indiscretions of other people.',\n", + " 'media_common.quotation.subjects',\n", + " 'Scandal'],\n", + " ['m.0448y9t',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Animales y Mas Que Animales', 'book.written_work.author', 'Saki'],\n", + " ['Columnist', 'common.topic.notable_types', 'Profession'],\n", + " ['When William Came', 'book.book.editions', 'When William came'],\n", + " ['Writer', 'people.profession.specializations', 'Folklorist'],\n", + " ['Novelist', 'base.schemastaging.context_name.pronunciation', 'g.125_kxbmc'],\n", + " ['Writer', 'people.profession.specializations', \"Children's writer\"],\n", + " ['Dante Alighieri', 'people.person.profession', 'Writer'],\n", + " ['Author',\n", + " 'tv.tv_subject.tv_episodes',\n", + " '\\\\\"The Power Of Love\\\\\" - Author - Robert Ernest Price'],\n", + " ['Playwright', 'base.descriptive_names.names.descriptive_name', 'm.0_z6m8v'],\n", + " ['m.0blp5sr', 'biology.hybrid_parentage.parent_sex', 'Male'],\n", + " ['Place of birth', 'type.property.schema', 'Person'],\n", + " [\"It's no use growing older if you only learn new ways of misbehaving yourself.\",\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['Intrusos y Otros Cuentos, Los', 'common.topic.notable_for', 'g.125f_zc0n'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010gbnxn'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Julia Child'],\n", + " ['Logs [Window]', 'film.film.film_set_decoration_by', 'Martins Grantovskis'],\n", + " ['m.0cgsz6j',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'George Sand'],\n", + " ['Authors Frequently Mentioned on the Web',\n", + " 'book.written_work.subjects',\n", + " 'Walt Whitman'],\n", + " ['Author',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Richard Castle'],\n", + " ['Bard', 'people.profession.specialization_of', 'Writer'],\n", + " ['Jorge Luis Borges',\n", + " 'influence.influence_node.influenced_by',\n", + " 'James Joyce'],\n", + " ['Reginald', 'book.book.editions', 'Reginald'],\n", + " ['Writer', 'people.profession.specializations', 'Gossip columnist'],\n", + " ['Stephen Fry', 'people.person.languages', 'English Language'],\n", + " ['Lyricist',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Christopher Hitchens',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Aldous Huxley'],\n", + " ['Contador de Historias, El', 'book.written_work.author', 'Saki'],\n", + " ['Republic of Ireland',\n", + " 'location.country.languages_spoken',\n", + " 'English Language'],\n", + " ['Author', 'type.type.expected_by', 'Author/editor'],\n", + " ['Author(s)', 'rdf-schema#range', 'Author'],\n", + " ['The Playboy of the Weekend World',\n", + " 'common.topic.notable_for',\n", + " 'g.125d9b8tb'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'william archer a fan of oscar wilde'],\n", + " ['Plato', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Humor Horror and the Supernatural', 'common.topic.notable_types', 'Book'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Benign prostatic hyperplasia'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced_by', 'Walter Pater'],\n", + " ['Intrusos y Otros Cuentos, Los', 'book.written_work.author', 'Saki'],\n", + " ['Date of death', 'type.property.schema', 'Deceased Person'],\n", + " ['Sredni Vashtar', 'film.film.other_crew', 'm.0nfgyzh'],\n", + " ['Date of burial', 'rdf-schema#domain', 'Deceased Person'],\n", + " ['W. H. Pugmire', 'people.person.gender', 'Male'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1vj8'],\n", + " ['The Happy Cat', 'common.topic.notable_types', 'Book'],\n", + " ['Samuel Beckett', 'people.person.nationality', 'Republic of Ireland'],\n", + " ['England', 'location.location.containedby', 'United Kingdom'],\n", + " ['Author(s)',\n", + " 'type.property.reverse_property',\n", + " 'Series Written (or Contributed To)'],\n", + " ['Writer',\n", + " 'common.topic.subject_of',\n", + " 'The Passion-Driven Writer and the Digital-Age Literary Marketplace'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " \"Great Socialist statesmen aren't made, they're still-born.\"],\n", + " ['Conrad the Wise', 'film.film.other_crew', 'm.0j5g7m8'],\n", + " ['walt whitman a fan was oscar wilde',\n", + " 'base.kwebbase.kwconnection.subject',\n", + " 'Walt Whitman'],\n", + " ['When William Came (Large Print)',\n", + " 'book.book_edition.publisher',\n", + " 'Echo Library'],\n", + " ['Saki', 'book.author.works_written', \"L'insupportable Bassington\"],\n", + " ['John Keats', 'people.person.nationality', 'England'],\n", + " ['m.03p8d5j', 'education.education.major_field_of_study', 'Classics'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'However, when the book edition appeared the following year it drew praise from Yeats In 1891 Wilde again traveled to Paris to work on a play entitled Salomé.'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y95'],\n", + " ['Authors Frequently Mentioned on the Web',\n", + " 'common.topic.image',\n", + " 'Authors Frequently Mentioned on the Web1.jpg'],\n", + " ['Writer', 'book.book_subject.works', 'Writers and Personality'],\n", + " ['Saki', 'people.person.place_of_birth', 'Sittwe'],\n", + " ['Saki', 'book.author.works_written', 'Der Almanach'],\n", + " ['Deceased Person', 'freebase.type_profile.strict_included_types', 'Person'],\n", + " ['Author', 'common.topic.subject_of', '\\\\\"Strickly O.G\\' Remind-U'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Stroke'],\n", + " ['When William Came (Large Print Edition)',\n", + " 'common.topic.notable_for',\n", + " 'g.125601r7r'],\n", + " ['Kilgore Trout',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Author'],\n", + " ['Playwright', 'base.descriptive_names.names.descriptive_name', 'm.0_z6mhb'],\n", + " ['Julius Evola', 'common.topic.notable_types', 'Author'],\n", + " ['Author',\n", + " 'book.book_subject.works',\n", + " 'The Passion-Driven Writer and the Digital-Age Literary Marketplace'],\n", + " ['Victor Hugo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Sredni Vashtar', 'film.film.other_crew', 'm.0nfg_bw'],\n", + " ['William Shakespeare', 'common.topic.notable_types', 'Author'],\n", + " ['Paul Merton', 'people.person.profession', 'Author'],\n", + " ['Author', 'type.type.expected_by', 'Editor'],\n", + " [\"C-SPAN's LCV 2011 U.S. Cities Tour Wraps up Savannah Shoot, Now in Charleston\",\n", + " 'book.written_work.subjects',\n", + " 'Author'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1w6s'],\n", + " ['Paris', 'location.administrative_division.country', 'France'],\n", + " ['m.0448yfx',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Abdominal aortic aneurysm', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Leprosy'],\n", + " ['Oscar Wilde', 'people.person.spouse_s', 'm.0s93mlt'],\n", + " ['Author', 'tv.tv_subject.tv_episodes', 'Pollee Freier - author'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.010fm14s'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'Carson his former friend and schoolmate from Trinity College, represented Queensberry.'],\n", + " ['Horror', 'media_common.media_genre.parent_genre', 'Speculative fiction'],\n", + " ['Writer', 'people.profession.specializations', 'Speculative writer'],\n", + " ['The novel and plays of Saki', 'book.written_work.author', 'Saki'],\n", + " ['Tooth and Claw', 'book.written_work.author', 'Saki'],\n", + " ['m.0yd6jxq',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Virginia Woolf', 'people.person.nationality', 'England'],\n", + " ['Morrissey', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " \"Lady Windermere's Fan\"],\n", + " [\"Jun'ichirō Tanizaki\", 'people.person.gender', 'Male'],\n", + " ['Author', 'base.schemastaging.context_name.pronunciation', 'g.125_lt3sr'],\n", + " ['Conrad the Wise', 'common.topic.notable_for', 'g.1255t24nf'],\n", + " ['Novelist',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Writers and Authors'],\n", + " ['Amyotrophic lateral sclerosis', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Dorian Gray', 'common.topic.notable_types', 'Film'],\n", + " ['Stephen Fry', 'people.person.profession', 'Journalist'],\n", + " ['William Godwin', 'people.person.nationality', 'United Kingdom'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville Ghost (Usborne Young Reading: Series Two)'],\n", + " ['No one has ever said it, but how painfully true it is that the poor have us always with them.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['The Complete Saki', 'common.topic.notable_types', 'Book'],\n", + " ['Sredni Vashtar', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['When William Came', 'book.written_work.subjects', 'Literary'],\n", + " ['m.0448xhz',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['m.0w4hkq8', 'projects.project_participation.role', 'Author'],\n", + " ['Male', 'common.topic.image', 'male.jpg'],\n", + " ['Incredible tales', 'common.topic.notable_for', 'g.12595vntg'],\n", + " ['W. H. Pugmire', 'common.topic.notable_types', 'Author'],\n", + " ['Writer', 'people.profession.specializations', 'Copywriter'],\n", + " ['Beaumont-Hamel', 'location.location.geolocation', 'm.02_gx5x'],\n", + " ['Oscar Wilde', 'people.person.profession', 'Poet'],\n", + " ['Wilde', 'common.topic.notable_types', 'Film'],\n", + " ['Animales y Mas Que Animales', 'common.topic.notable_types', 'Book'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Abdominal aortic aneurysm'],\n", + " ['Oscar Wilde', 'media_common.dedicator.dedications', 'm.04_m3tk'],\n", + " ['m.0j5g7wt', 'film.performance.film', 'Conrad the Wise'],\n", + " ['Journalist',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Edmund Grey'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1wb0'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1vy1'],\n", + " ['m.04_m3tk', 'media_common.dedication.dedicated_by', 'Oscar Wilde'],\n", + " ['Author',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " 'Very little in science fiction can transcend the gimmickry of a technical conceit, yet without that conceit at its heart a book is not truly science fiction. Furthermore, so little emerging thought and technology is employed by sf writers today that the genre is lagging far behind reality both in the cosmology area and the technology area: sf is no longer a place to experiment, but is now very derivative.'],\n", + " ['Kilgore Trout',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Writer'],\n", + " ['Cardiovascular disease',\n", + " 'medicine.disease_cause.diseases',\n", + " 'Transient ischemic attack'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville ghost'],\n", + " ['When William Came', 'book.book.editions', 'When William Came'],\n", + " ['Oscar Wilde', 'book.author.book_editions_published', 'Canterville Ghost'],\n", + " ['The Unbearable Bassington',\n", + " 'book.book.editions',\n", + " 'The Unbearable Bassington'],\n", + " ['We all know that Prime Ministers are wedded to the truth, but like other wedded couples they sometimes live apart.',\n", + " 'media_common.quotation.subjects',\n", + " 'Politicians and Politics'],\n", + " ['Hector Hugh Munro', 'common.image.size', 'm.02bjf_w'],\n", + " ['Myanmar', 'location.country.administrative_divisions', 'Rakhine State'],\n", + " ['Morrissey', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville Ghost (Treasure)'],\n", + " ['Oscar Wilde', 'award.award_nominee.award_nominations', 'm.0ztr7vv'],\n", + " ['Logs [Window]', 'film.film.other_crew', 'm.0pbst48'],\n", + " ['Playwright',\n", + " 'base.descriptive_names.names.descriptive_name',\n", + " 'm.01066t9b'],\n", + " ['Virginia Woolf', 'freebase.valuenotation.is_reviewed', 'Date of death'],\n", + " ['Sittwe', 'location.location.people_born_here', 'Saki'],\n", + " ['The Westminster Alice', 'book.book.genre', 'Fantasy'],\n", + " ['m.03pfwzk', 'people.place_lived.person', 'Oscar Wilde'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The picture of Dorian Gray'],\n", + " ['heart attack',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Abdominal aortic aneurysm'],\n", + " ['Author', 'type.type.expected_by', 'Editor of this edition'],\n", + " ['Author', 'type.type.expected_by', 'Contributing authors'],\n", + " ['m.0ztr7jq', 'award.award_honor.honored_for', 'The Picture of Dorian Gray'],\n", + " ['Reginald', 'common.topic.notable_for', 'g.12567ds1g'],\n", + " ['Christopher Hitchens',\n", + " 'people.deceased_person.cause_of_death',\n", + " 'Esophageal cancer'],\n", + " ['m.07n73w_',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Author', 'book.book_subject.works', 'Fame & Folly'],\n", + " ['Dave Eggers', 'people.person.profession', 'Novelist'],\n", + " ['Author', 'base.descriptive_names.names.descriptive_name', 'm.0_zdcrs'],\n", + " ['Film', 'freebase.type_profile.published', 'Published'],\n", + " ['m.0448yg8',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0448y9t'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'His outrageous behavior, which brought him public attention, linked his name to the British Aesthetic movement and eventually led to his downfall, when his homosexuality was exposed.'],\n", + " ['Conrad the Wise', 'film.film.genre', 'Short Film'],\n", + " ['Literature',\n", + " 'book.school_or_movement.associated_works',\n", + " 'Authors Frequently Mentioned on the Web'],\n", + " ['m.0pbssx7', 'film.film_crew_gig.film', 'Logs [Window]'],\n", + " [\"Children with Hyacinth's temperament don't know better as they grow older; they merely know more.\",\n", + " 'common.topic.notable_types',\n", + " 'Quotation'],\n", + " ['The sacrifices of friendship were beautiful in her eyes as long as she was not asked to make them.',\n", + " 'media_common.quotation.subjects',\n", + " 'Friends and Friendship'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Epilepsy'],\n", + " ['Oscar Wilde', 'base.americancomedy.comedian.comedy_genres', 'Wit'],\n", + " ['Conrad the Wise', 'film.film.release_date_s', 'm.0j5g79z'],\n", + " ['James K. Morrow', 'people.person.profession', 'Novelist'],\n", + " ['William Godwin',\n", + " 'influence.influence_node.influenced_by',\n", + " 'William Shakespeare'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0_f_41y'],\n", + " ['Picardy', 'location.administrative_division.country', 'France'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Alcohol abuse'],\n", + " ['Sredni Vashtar', 'film.film.starring', 'm.0nf31xf'],\n", + " ['Oscar Wilde', 'award.award_winner.awards_won', 'm.0ztr7jq'],\n", + " ['Massimo Zanini', 'freebase.domain_profile.category', 'Male'],\n", + " ['English Language', 'language.human_language.main_country', 'Canada'],\n", + " ['Oscar Wilde', 'people.person.parents', 'Jane Wilde'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1wdc'],\n", + " ['Victor Hugo', 'people.person.profession', 'Writer'],\n", + " ['Logs [Window]', 'film.film.film_casting_director', 'Inese Bikse'],\n", + " ['When William Came (Large Print)',\n", + " 'common.topic.notable_types',\n", + " 'Consumer product'],\n", + " ['Author',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '\\\\\"It’s a good thing that he [John Galsworthy] and those who answered his call in the twentieth century have given the rest of us tilling fields in the twenty-first century a 90-year head start.\\\\\"'],\n", + " ['Edgar Allan Poe',\n", + " 'fictional_universe.fictional_character.based_on',\n", + " 'Edgar Allan Poe'],\n", + " ['Oscar Wilde',\n", + " 'book.author.book_editions_published',\n", + " 'The Canterville Ghost (English Easy Readers)'],\n", + " ['Wilde', 'media_common.netflix_title.netflix_genres', 'Biography'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010b1v6n'],\n", + " ['Essayist', 'people.profession.specialization_of', 'Writer'],\n", + " ['English Language',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'England'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " \"However, Queensberry's lawyers turned over all their evidence to the Director of Public Prosecutions.\"],\n", + " [\"Jun'ichirō Tanizaki\", 'people.person.profession', 'Novelist'],\n", + " ['Playwright', 'base.descriptive_names.names.descriptive_name', 'm.0_z8pkn'],\n", + " ['Irish people', 'people.ethnicity.languages_spoken', 'English Language'],\n", + " ['Travel writer', 'people.profession.specialization_of', 'Writer'],\n", + " ['m.02wngvx', 'education.education.student', 'Oscar Wilde'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.has_sentences',\n", + " 'He was now bankrupt and permanently separated from his wife and children.'],\n", + " ['Victor Hugo', 'people.person.profession', 'Novelist'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.0106cxg8'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Julius Evola'],\n", + " ['Phone Trick', 'biology.organism.sex', 'Male'],\n", + " ['Saki',\n", + " 'people.person.quotations',\n", + " 'He spends his life explaining from his pulpit that the glory of Christianity consists in the fact that though it is not true it has been found necessary to invent it.'],\n", + " [\"You needn't tell me that a man who doesn't love oysters and asparagus and good wines has got a soul, or a stomach either. He's simply got the instinct for being unhappy highly developed.\",\n", + " 'media_common.quotation.subjects',\n", + " 'Food and Eating'],\n", + " ['No one can be an unbeliever nowadays. The Christian Apologists have left one nothing to disbelieve.',\n", + " 'media_common.quotation.author',\n", + " 'Saki'],\n", + " ['Paul Merton', 'influence.influence_node.influenced_by', 'Oscar Wilde'],\n", + " ['When William Came', 'common.topic.article', 'm.0dby3q'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Agent'],\n", + " ['m.0j5g7m4', 'film.film_crew_gig.film', 'Conrad the Wise'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'oscar wilde met camille pissarro'],\n", + " ['Sittwe', 'common.topic.image', 'Sittwe main street'],\n", + " ['Saki', 'book.author.works_written', 'A Saki sampler'],\n", + " ['Reginald in Russia and Other Sketches',\n", + " 'book.book.genre',\n", + " 'Speculative fiction'],\n", + " ['m.0nf32s9', 'film.performance.film', 'Sredni Vashtar'],\n", + " [\"Great Socialist statesmen aren't made, they're still-born.\",\n", + " 'common.topic.notable_for',\n", + " 'g.1255l1324'],\n", + " ['Playwright', 'rdf-schema#range', 'Author'],\n", + " ['m.0bnmqyj', 'base.prison.imprisonment.prisoner', 'Oscar Wilde'],\n", + " ['m.08dd9w4', 'award.award_honor.honored_for', 'Sredni Vashtar'],\n", + " ['Novelist', 'base.descriptive_names.names.descriptive_name', 'm.010fp21b'],\n", + " ['Jorge Luis Borges', 'people.person.profession', 'Writer'],\n", + " ['Author',\n", + " 'base.skosbase.vocabulary_equivalent_topic.narrower_concept',\n", + " 'Authorship'],\n", + " ['Author', 'common.topic.subject_of', 'Salvatore Capolupo'],\n", + " ['m.09ybk14', 'common.webpage.topic', 'Oscar Wilde'],\n", + " ['Saki', 'book.author.works_written', 'Juguetes de Paz - El Huevo Cuadrado'],\n", + " ['Oscar Wilde',\n", + " 'base.kwebbase.kwtopic.connections_to',\n", + " 'james abbott mcneill whistler a fan was oscar wilde'],\n", + " ['James K. Morrow', 'common.topic.notable_types', 'Author'],\n", + " ['Victor Hugo', 'influence.influence_node.influenced', 'Jorge Luis Borges'],\n", + " ['Humor Horror and the Supernatural',\n", + " 'book.book.genre',\n", + " 'Speculative fiction'],\n", + " ['Oscar Wilde',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'The Importance of Being Earnest'],\n", + " ['Sredni Vashtar', 'film.film.release_date_s', 'm.0nf2pdl'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced_by', 'John Ruskin'],\n", + " ['Sredni Vashtar', 'film.film.story_by', 'Saki'],\n", + " ['Satirist', 'people.profession.corresponding_type', 'Author'],\n", + " ['Oscar Wilde', 'influence.influence_node.influenced', 'Dave Sim'],\n", + " ['m.04zv8gw',\n", + " 'comic_strips.comic_strip_creator_duration.creator_role',\n", + " 'Writer'],\n", + " ['Paul Merton', 'people.person.religion', 'Catholicism'],\n", + " ['Victor Hugo',\n", + " 'base.kwebbase.kwtopic.connections_from',\n", + " 'victor hugo met oscar wilde'],\n", + " ['Aldous Huxley', 'people.person.profession', 'Writer'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Inguinal hernia'],\n", + " ['Jorge Luis Borges',\n", + " 'influence.influence_node.influenced',\n", + " 'Alejandro Dolina'],\n", + " ['Epilepsy', 'medicine.disease.risk_factors', 'Meningitis'],\n", + " ['Writer', 'comic_strips.comic_strip_creator_role.creators', 'm.0g9p00h'],\n", + " ['When William Came', 'common.topic.notable_for', 'g.1255sxw3g'],\n", + " ...],\n", + " [['m.02_1x1x',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['George Weasley',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.0j8zc0j',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['When you have seen as much of life as I have, you will not underestimate the power of obsessive love.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.08f5br7'],\n", + " ['J. K. Rowling', 'people.person.places_lived', 'm.04fkq8n'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'media_common.quotation_source.quotations',\n", + " \"I sometimes find, and I am sure you know the feeling, that I simply have too many thoughts and memories crammed into my mind.... At these times... I use the Pensieve. One simply siphons the excess thoughts from one's mind, pours them into the basin, and examines them at one's leisure.\"],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Diagon Alley'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Lord Voldemort'],\n", + " ['Selwyn',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.0h4z_w_'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.written_work.subjects',\n", + " 'England'],\n", + " ['Where your treasure is, there will your heart be also.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Lucius Malfoy', 'common.topic.image', 'LuciusMalfoy.jpg'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Maria Ushiromiya'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'common.topic.webpage',\n", + " 'm.09xx1mt'],\n", + " ['Time is making fools of us again.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Rita Skeeter',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic'],\n", + " ['Harry Potter and the Deathly Hallows - Part I',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Deathly Hallows: Part I'],\n", + " ['Lord Voldemort', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Character', 'owl#inverseOf', 'Portrayed in films'],\n", + " [\"We are only as strong as we are united, as weak as we are divided. Lord Voldemort's gift for spreading discord and enmity is very great. We can fight it only by showing an equally strong bond of friendship and trust. Differences of habit and language are nothing at all if our aims are identical and our hearts are open.\",\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter e la Camera Dei Segreti'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Arthur Weasley'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Transient ischemic attack'],\n", + " ['Merope Gaunt',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " \"Salazar Slytherin's Locket\"],\n", + " ['Lord Voldemort',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Neville Longbottom',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'common.topic.notable_for',\n", + " 'g.125dz6zlg'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter E la Camera Dei Segreti'],\n", + " ['Amycus Carrow',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'The mind is not a book, to be opened at will and examined at leisure. Thoughts are not etched on the inside of skulls, to be perused by an invader. The mind is a complex and many-layered thing.'],\n", + " ['Cedric Diggory', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " [\"Marvolo Gaunt's Ring\", 'common.topic.notable_types', 'Fictional Object'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " \"Rowena Ravenclaw's Diadem\"],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.written_work.next_in_series',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.099sqv4'],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Rodolphus Lestrange'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gout'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Tinworth'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Fantasy'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Barrett's esophagus\"],\n", + " [\"Dumbledore's Army\",\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.122mr0bj'],\n", + " ['Abdominal aortic aneurysm',\n", + " 'medicine.disease.parent_disease',\n", + " 'Cardiovascular disease'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Prostate cancer'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'media_common.adaptation.adapted_from',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'James Potter'],\n", + " ['Lord Voldemort',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Draco Malfoy', 'film.film_character.portrayed_in_films', 'm.075nc31'],\n", + " ['Elder Wand', 'fictional_universe.fictional_object.owner', 'Gregorovitch'],\n", + " ['Elder Wand',\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Antioch Peverell'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter och de vises sten'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.0b46q_k'],\n", + " ['Author', 'people.profession.specialization_of', 'Writer'],\n", + " ['Gellert Grindelwald',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['You place too much importance... on the so-called purity of blood! You fail to recognize that it matters not what someone is born, but what they grow to be!',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the deathly hallows'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'The Bloody Baron'],\n", + " ['Cho Chang', 'common.topic.notable_types', 'Film character'],\n", + " ['m.0h371wb',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Malfoy Manor'],\n", + " ['Dobby the House Elf',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Deathly Hallows (Book 7)',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Deathly Hallows: Part I',\n", + " 'cvg.computer_videogame.sequel',\n", + " 'Harry Potter and the Deathly Hallows – Part 2'],\n", + " ['Ministry of Magic',\n", + " 'fictional_universe.fictional_setting.universe',\n", + " 'Harry Potter'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'media_common.quotation_source.quotations',\n", + " \"You are here to learn the subtle science and exact art of potion-making. As there is little foolish wand-waving here, many of you will hardly believe this is magic. I don't expect you will really understand the beauty of the softly simmering cauldron with its shimmering fumes, the delicate power of liquids that creep through human veins, bewitching the mind, ensnaring the senses -- I can teach you how to bottle fame, brew glory, even stopper death -- if you aren't as big a bunch of dunderheads as I usually have to teach.\"],\n", + " ['J. K. Rowling',\n", + " 'influence.influence_node.influenced_by',\n", + " 'J. D. Salinger'],\n", + " ['Lucius Malfoy',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Arabella Figg'],\n", + " ['Rose Weasley',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Michael Corner'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " 'Cloak of invisibility'],\n", + " ['Tom Felton', 'film.actor.film', 'm.0h0xnc6'],\n", + " ['Pomona Sprout',\n", + " 'fictional_universe.fictional_character.employers',\n", + " 'm.091fn5s'],\n", + " ['m.0x2l5lg',\n", + " 'award.award_nomination.nominated_for',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Harry Potter and the Goblet of Fire', 'book.book.characters', 'Goyle Sr.'],\n", + " ['Bellatrix Lestrange',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Death Eaters'],\n", + " ['Epilepsy', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'common.topic.article',\n", + " 'm.014jt3'],\n", + " ['J. K. Rowling', 'book.author.works_written', 'Llaves Voladoras, Las'],\n", + " ['George Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " [\"Dumbledore's Army\",\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.11b6pkl55c'],\n", + " ['Lord Voldemort',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " 'Elder Wand'],\n", + " ['heart attack',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Harry Potter',\n", + " 'media_common.quotation_addressee.quotations',\n", + " 'Fear of a name increases fear of the thing itself.'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Fantasy'],\n", + " ['Augustus Rookwood',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Lord Voldemort',\n", + " 'base.fictionaluniverse.fictional_killer.characters_killed',\n", + " 'James Potter'],\n", + " ['Harry Potter and the Prisoner of Azkaban', 'book.book.genre', 'Fantasy'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_setting.contains',\n", + " \"Hagrid's Hut\"],\n", + " ['Percy Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Death Eaters', 'common.topic.article', 'm.014k9k'],\n", + " ['Rita Skeeter',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'England'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.07n73w_'],\n", + " ['m.09xhhcd',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.09xtg5f',\n", + " 'common.webpage.topic',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'film.film.starring',\n", + " 'm.075nc2t'],\n", + " ['Jimmy Peakes',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor House'],\n", + " ['m.0b47c6z', 'common.webpage.in_index', 'WSJ Speakeasy Index'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Horklump'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'g.1ym_l5qwv'],\n", + " ['Professor Horace Slughorn',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Wizarding world'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0j7v__g'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Gryffindor Quidditch Team',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Dean Thomas'],\n", + " [\"Parkinson's disease\",\n", + " 'medicine.disease.symptoms',\n", + " 'Seborrheic dermatitis'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Vincent Crabbe'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Charlie Weasley'],\n", + " ['m.02wm0c5',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Narcissa Malfoy', 'common.topic.notable_for', 'g.12557t6pq'],\n", + " ['m.02_98hd',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Harry Potter and the Deathly Hallows: Part I',\n", + " 'media_common.adaptation.adapted_from',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'Bigotry is probably the thing I detest most.'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0x2l5lg'],\n", + " ['Draco Malfoy', 'film.film_character.portrayed_in_films', 'm.09lybdb'],\n", + " ['m.0w0_dk9', 'film.performance.character', 'Lucius Malfoy'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Lord Voldemort'],\n", + " ['Harry Potter and the Deathly Hallows (Book 7) (Library Edition)',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['m.0sgkpw5', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Hermione Granger',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Sirius Black'],\n", + " ['United Kingdom', 'location.country.first_level_divisions', 'England'],\n", + " ['Harry Potter literary series',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_sh9gg'],\n", + " ['Harry Potter', 'internet.website_category.sites', 'Harry Potter Fan Zone'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Jobberknoll'],\n", + " ['Rita Skeeter', 'common.topic.notable_types', 'Film character'],\n", + " ['m.09yx34b',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Gryffindor Quidditch Team',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Ginny Weasley'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Ronald Weasley'],\n", + " ['Harry Potter and the Chamber of Secrets', 'film.film.genre', 'Fantasy'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.written_work.original_language',\n", + " 'English Language'],\n", + " ['G. K. Chesterton', 'common.topic.notable_types', 'Author'],\n", + " ['Vincent Crabbe', 'common.topic.notable_types', 'Film character'],\n", + " [\"Marauders' Map\",\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Fred Weasley'],\n", + " ['Ted Lupin',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Nymphadora Lupin'],\n", + " ['Male', 'common.topic.article', 'm.05zpq8'],\n", + " ['m.0b48twx', 'common.webpage.in_index', 'WSJ Speakeasy Index'],\n", + " ['Young-adult fiction', 'media_common.media_genre.parent_genre', 'Fiction'],\n", + " ['J. K. Rowling', 'people.person.parents', 'Peter Rowling'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0sgp7np'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Peeves'],\n", + " ['J. K. Rowling', 'award.award_winner.awards_won', 'm.0mwlc80'],\n", + " ['Rufus Scrimgeour',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.0sgp7np',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter literary series'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date written'],\n", + " ['Professor Minerva McGonagall',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Snidget'],\n", + " ['George Weasley',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor House'],\n", + " ['Professor Albus Dumbledore',\n", + " 'common.topic.notable_types',\n", + " 'Film character'],\n", + " ['George Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Tom Felton', 'film.actor.film', 'm.075nc2t'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['m.09wz4p1',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Nearly Headless Nick'],\n", + " ['Fat Friar',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.genre',\n", + " \"Children's literature\"],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'Curiosity is not a sin.... But we should exercise caution with our curiosity... yes, indeed.'],\n", + " ['Lucius Malfoy',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Draco Malfoy'],\n", + " ['Regulus Black', 'common.topic.notable_types', 'Film character'],\n", + " ['Eileen Prince',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.09vqd0m'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.editions',\n", + " 'Harry Potter Y El Caliz De Fuego'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.written_work.previous_in_series',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['m.0b47c6z',\n", + " 'common.webpage.topic',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Deathly Hallows (Book 7)'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Sybill Trelawney'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.sub_organization_in_fiction',\n", + " 'Slytherin Quidditch Team'],\n", + " ['Wizarding world',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['m.098g8_1', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['m.0gkbzhc', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['m.0h0xnc6', 'film.performance.actor', 'Tom Felton'],\n", + " ['Hermione Granger',\n", + " 'fictional_universe.fictional_object_destroyer.fictional_objects_destroyed',\n", + " \"Hufflepuff's cup\"],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_object_destroyer.fictional_objects_destroyed',\n", + " \"Tom Riddle's diary\"],\n", + " ['Parents', 'type.property.reverse_property', 'Children'],\n", + " ['Scotland', 'base.biblioness.bibs_location.country', 'United Kingdom'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " \"Florean Fortescue's Ice Cream Parlour\"],\n", + " ['The Slug Club',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Regulus Black'],\n", + " ['It takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['m.09xyydk', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['T. H. White', 'influence.influence_node.influenced', 'J. K. Rowling'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.02vhpyb'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Fred Weasley'],\n", + " ['Gellert Grindelwald', 'common.topic.notable_types', 'Book Character'],\n", + " ['Harry Potter and the deathly Hallows',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Oliver Woods',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor Quidditch Team'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " \"Godric's Hollow\"],\n", + " ['Vernon Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Rose Weasley', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'media_common.quotation_source.quotations',\n", + " 'People find it far easier to forgive others for being wrong than being right.'],\n", + " ['Harry Potter',\n", + " 'freebase.equivalent_topic.equivalent_domain',\n", + " 'Harry Potter Fanbase'],\n", + " ['Petunia Evans Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Gryffindor Quidditch Team',\n", + " 'fictional_universe.fictional_organization.appears_in_universes',\n", + " 'Harry Potter'],\n", + " ['Professor Minerva McGonagall',\n", + " 'fictional_universe.fictional_character.employers',\n", + " 'm.02wm0cg'],\n", + " ['Secretum Secretorum',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Date of first publication'],\n", + " ['Elder Wand', 'fictional_universe.fictional_object.owner', 'Harry Potter'],\n", + " ['Frank Bryce',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney Stone'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['J. K. Rowling',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0j3q_dn'],\n", + " ['C. S. Lewis', 'common.topic.notable_types', 'Author'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['m.09xhhbz', 'common.webpage.topic', 'Harry Potter and the Goblet of Fire'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'The consequences of our actions are so complicated, so diverse, that predicting the future is a very difficult business indeed.'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " \"If you're holding out for universal popularity, I'm afraid you will be in this cabin for a very long time.\"],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0z1rm_7'],\n", + " ['Cedric Diggory',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Hufflepuff House'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0b6997_'],\n", + " ['Frank Bryce',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['J. K. Rowling', 'people.person.spouse_s', 'm.04fkqf3'],\n", + " ['Ah, music. A magic beyond all we do here!',\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'g.122mr0bj'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.written_work.subjects',\n", + " 'Schools'],\n", + " ['Fred Weasley',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Arthur Weasley'],\n", + " ['m.09xsnpz',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Remus Lupin',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Sirius Black'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d521lm'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Deathly Hallows (Book 7)'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Murtlap'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Cedric Diggory'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Diagon Alley'],\n", + " ['The Slug Club',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Harry Potter'],\n", + " ['Arthur Weasley',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Percy Weasley'],\n", + " ['The Slug Club',\n", + " 'fictional_universe.fictional_organization.fictional_organization_founder',\n", + " 'Professor Horace Slughorn'],\n", + " ['Leprechaun',\n", + " 'fictional_universe.character_species.found_in_fictional_universe',\n", + " 'Harry Potter'],\n", + " ['Vernon Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['m.09ytz_f',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.04fkqg7', 'people.place_lived.person', 'J. K. Rowling'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Neville Longbottom'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'film.film.story_by',\n", + " 'J. K. Rowling'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Charlie Weasley'],\n", + " ['Bellatrix Lestrange',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Female'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter et la Chambre des Secrets'],\n", + " ['Scotland', 'location.administrative_division.country', 'United Kingdom'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.editions',\n", + " \"Harry Potter Et L'Ordre Du Phenix\"],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0gk9sds'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " \"Marvolo Gaunt's Ring\"],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0bj8y_v'],\n", + " ['Peeves',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['m.0_sh9gg', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Rubeus Hagrid'],\n", + " ['m.0hj5cm7',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.parent_organization_in_fiction',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Charlie Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " [\"Marvolo Gaunt's Ring\",\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Professor Albus Dumbledore'],\n", + " ['m.0sgpcz0', 'award.award_honor.award_winner', 'J. K. Rowling'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.organizations',\n", + " 'Gryffindor House'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'Age is foolish and forgetful when it underestimates youth.'],\n", + " ['J. K. Rowling', 'book.author.works_written', \"The Cuckoo's Calling\"],\n", + " ['Jimmy Peakes',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Ronald Weasley',\n", + " 'fictional_universe.fictional_object_destroyer.fictional_objects_destroyed',\n", + " \"Salazar Slytherin's Locket\"],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.122bry5k'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Augurey'],\n", + " ['m.0hh6353',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Rubeus Hagrid',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Draco Malfoy',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Occlumency'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " 'Harrius Potter et Philosophi Lapis (Latin Edition)'],\n", + " ['Film character', 'freebase.type_profile.kind', 'Annotation'],\n", + " ['Draco Malfoy',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['m.0bggzmc', 'common.webpage.topic', 'Narcissa Malfoy'],\n", + " ['Sybill Trelawney',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Female'],\n", + " ['Remus Lupin',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.09yv55b', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Graphorn'],\n", + " ['Mrs. Norris',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['m.09xhhbz',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.employers',\n", + " 'm.02vhr7_'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['m.0sgktwf', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " \"Harry Potter and the philosopher's stone.\"],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book.interior_illustrations_by',\n", + " 'Mary GrandPré'],\n", + " ['m.09xtg5f',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Gryffindor House',\n", + " 'common.topic.notable_types',\n", + " 'Organization in fiction'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'media_common.quotation_source.quotations',\n", + " \"We are only as strong as we are united, as weak as we are divided. Lord Voldemort's gift for spreading discord and enmity is very great. We can fight it only by showing an equally strong bond of friendship and trust. Differences of habit and language are nothing at all if our aims are identical and our hearts are open.\"],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Ravenclaw Tower'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Sir Patrick Delaney-Podmore'],\n", + " ['Witchcraft',\n", + " 'film.film_subject.films',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter Aur Azkaban Ka Qaidi'],\n", + " ['Professor Horace Slughorn',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Date of first publication'],\n", + " ['E. Nesbit', 'influence.influence_node.influenced', 'J. K. Rowling'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Olympe Maxime'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Death Eaters', 'common.topic.webpage', 'm.09y14rb'],\n", + " ['Fiction', 'media_common.media_genre.child_genres', 'Young-adult fiction'],\n", + " ['m.09w6w45', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'g.121v85j3'],\n", + " ['m.04fkpwn', 'people.sibling_relationship.sibling', 'J. K. Rowling'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " [\"Dumbledore's Army\",\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Cho Chang'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'g.11b6b7c50g'],\n", + " ['Fleur Delacour',\n", + " 'fictional_universe.fictional_character.species',\n", + " 'Veela'],\n", + " ['Remus Lupin',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Regulus Black',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Half-Blood Prince', 'book.book.genre', 'Fantasy'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Dupuytren's contracture\"],\n", + " ['m.0853_tn', 'award.award_honor.award_winner', 'Mary GrandPré'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Bellatrix Lestrange'],\n", + " ['Pomona Sprout',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Hufflepuff House'],\n", + " ['Harry Potter and the Deathly Hallows – Part 2',\n", + " 'film.film.produced_by',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Wizarding world'],\n", + " ['Walden Macnair',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Death Eaters'],\n", + " ['Firenze',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Dean Thomas',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " \"Dumbledore's Army\"],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Actinic keratosis'],\n", + " ['m.0csg5rx',\n", + " 'film.performance.film',\n", + " 'Harry Potter and the Deathly Hallows - Part I'],\n", + " ['Harry Potter', 'internet.website_category.sites', 'HPANA'],\n", + " ['Lucius Malfoy',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Mrs. Norris', 'common.topic.notable_types', 'Book Character'],\n", + " ['m.09ym013',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Professor Minerva McGonagall',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'The Slug Club'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Remus Lupin'],\n", + " ['J. K. Rowling', 'tv.tv_actor.starring_roles', 'm.0bv267t'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'film.film.sequel',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['m.02wm0bw',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Garri Potter i uznik Azkabana'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'film.film.prequel',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Ginny Weasley'],\n", + " ['Diagon Alley',\n", + " 'fictional_universe.fictional_setting.contains',\n", + " \"Madam Malkin's Robes For All Occasions\"],\n", + " ['It is our choices...that show what we truly are, far more than our abilities.',\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Moaning Myrtle'],\n", + " ['Harry Potter',\n", + " 'media_common.quotation_addressee.quotations',\n", + " 'It does not do to dwell on dreams and forget to live.'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " \"If you want to know what a man's like, take a good look at how he treats his inferiors, not his equals.\"],\n", + " ['It is my belief... that the truth is generally preferable to lies.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Percy Weasley',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.written_work.subjects',\n", + " 'Magic'],\n", + " ['Harry Potter and the Deathly Hallows - Part I',\n", + " 'film.film.genre',\n", + " 'Fantasy'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " \"Tom Riddle's diary\"],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Mrs. Norris'],\n", + " ['m.0h4z_w_',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Nymphadora Lupin',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter y La Camara Secreta - 2'],\n", + " ['Wizarding world',\n", + " 'fictional_universe.fictional_setting.universe',\n", + " 'Harry Potter'],\n", + " ['Gryffindor Quidditch Team',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Charlie Weasley'],\n", + " ['m.0853_tn',\n", + " 'award.award_honor.honored_for',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " \"Harry Potter et le Prisonnier d'Azkaban\"],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'You place too much importance... on the so-called purity of blood! You fail to recognize that it matters not what someone is born, but what they grow to be!'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Salamander'],\n", + " ['Rufus Scrimgeour', 'common.topic.notable_types', 'Film character'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Lily Evans Potter'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Bundimun'],\n", + " ['Gregorovitch',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['The mind is not a book, to be opened at will and examined at leisure. Thoughts are not etched on the inside of skulls, to be perused by an invader. The mind is a complex and many-layered thing.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.040vl1y'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'g.11b6nx6fxj'],\n", + " ['Wizarding world',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Remus Lupin', 'common.topic.notable_types', 'Film character'],\n", + " ['Michael Corner',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['James Potter',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Marauders'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_organization_founder.fictional_organizations_founded',\n", + " 'Order of the Phoenix'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'g.121315b0'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'film.film.sequel',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['m.0bgg_2v', 'common.webpage.category', 'Topic Webpage'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Hungarian Horntail Dragon'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Common Welsh Green Dragon'],\n", + " ['Vernon Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter Fanbase',\n", + " 'freebase.domain_profile.equivalent_topic',\n", + " 'Harry Potter'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09wz4p1'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09xnj6h'],\n", + " ['Ginny Weasley',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Holyhead Harpies'],\n", + " ['Sylvia Plath', 'people.person.gender', 'Female'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.written_work.subjects',\n", + " 'Literature'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0b699bn'],\n", + " ['Pancreatic cancer',\n", + " 'medicine.icd_9_cm_classification.parent_classification',\n", + " 'Pancreatic cancer'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'g.11b6pnz7bg'],\n", + " ['Narcissa Malfoy',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['J. K. Rowling', 'people.person.profession', 'Novelist'],\n", + " ['Male',\n", + " 'medicine.risk_factor.diseases',\n", + " 'Attention deficit hyperactivity disorder'],\n", + " ['Lucius Malfoy',\n", + " 'fictional_universe.fictional_character.married_to',\n", + " 'm.075hqy4'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.written_work.previous_in_series',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['It does not do to dwell on dreams and forget to live.',\n", + " 'media_common.quotation.source',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Ronald Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Stephen King', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.11b6pnz7bg'],\n", + " ['Ernie Macmillan',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'g.121yb32s'],\n", + " ['To the well-organized mind, death is but the next great adventure.',\n", + " 'media_common.quotation.source',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Mrs. Norris',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Happiness can be found, even in the darkest of times, if one only remembers to turn on the light.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0b699fj'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.written_work.subjects',\n", + " 'Good and evil'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Avery Jr.'],\n", + " ['Rubeus Hagrid',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Neville Longbottom'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.parent_cause_of_death',\n", + " 'Cardiovascular disease'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Sybill Trelawney'],\n", + " ['C. S. Lewis', 'people.person.profession', 'Writer'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter und der Gefangene von Azkaban'],\n", + " ['m.07ngrwb',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Ronald Weasley',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'The Burrow'],\n", + " [\"Dumbledore's Army Reunites at Quidditch World Cup Final\",\n", + " 'book.short_story.characters',\n", + " 'Ginny Weasley'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'George Weasley'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['It is true, however, that those who have mastered Legilimency are able, under certain conditions, to delve into the minds of their victims and to interpret their findings correctly....',\n", + " 'media_common.quotation.addressee',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Hermione Granger',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['m.0bgg_y8', 'common.webpage.topic', 'Draco Malfoy'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Gastritis'],\n", + " ['m.0b697tw', 'award.award_honor.award_winner', 'J. K. Rowling'],\n", + " ['Susan Bones',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Colin Creevey', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Mozenrath'],\n", + " ['m.0b699fj', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Yaxley'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Chimaera'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Sebaceous cyst'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_organization.fictional_organization_founder',\n", + " 'Salazar Slytherin'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'People find it far easier to forgive others for being wrong than being right.'],\n", + " ['George Eliot', 'people.person.gender', 'Female'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.121lgztv'],\n", + " ['m.0g29h5z', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['Molly Weasley',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'g.11b6rb871k'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.editions',\n", + " 'Harry Potter y el Caliz de Fuego'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Ronald Weasley'],\n", + " [\"Kreacher is what he has been made by wizards, Harry. Yes, he is to be pitied. His existence has been as miserable as your friend Dobby's. He was forced to do Sirius's bidding, because Sirius was the last of the family to which he was enslaved, but he felt no true loyalty to him. And whatever Kreacher's faults, it must be admitted that Sirius did nothing to make Kreacher's lot easier.\",\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'film.film.starring',\n", + " 'm.03lwx7f'],\n", + " ['m.0dl9g_3',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'fictional_universe.work_of_fiction.part_of_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.11b6mp5yt4'],\n", + " ['Andromeda Tonks',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Wizarding world',\n", + " 'fictional_universe.fictional_setting.works_set_here',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Fictional Universe', 'freebase.type_profile.published', 'Published'],\n", + " ['Ronald Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Remember, if the time should come when you have to make a choice between what is right, and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'fictional_universe.work_of_fiction.part_of_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Professor Minerva McGonagall',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Pomona Sprout'],\n", + " ['C. S. Lewis', 'people.person.nationality', 'United Kingdom'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Nott Sr.'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Ronald Weasley'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Professor Filius Flitwick'],\n", + " ['United Kingdom',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'England'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the deathly Hallows'],\n", + " ['J. K. Rowling',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0gyzbxr'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Andromeda Tonks'],\n", + " ['m.0jz6g1', 'film.performance.character', 'Lucius Malfoy'],\n", + " ['J. K. Rowling', 'common.topic.image', 'jkrowling.jpg'],\n", + " ['England',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United Kingdom'],\n", + " ['Phineas Nigellus Black',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Fred Weasley',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Narcissa Malfoy',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.122mr0bj'],\n", + " ['Molly Weasley',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Percy Weasley'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.099sqnh'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " \"We are only as strong as we are united, as weak as we are divided. Lord Voldemort's gift for spreading discord and enmity is very great. We can fight it only by showing an equally strong bond of friendship and trust. Differences of habit and language are nothing at all if our aims are identical and our hearts are open.\"],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Phineas Nigellus Black'],\n", + " ['m.0d0pw2n',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Ministry of Magic'],\n", + " ['m.0h0xnc6', 'film.performance.character', 'Draco Malfoy'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"HARRY POTTER AND THE PHILOSOPHER'S STONE.\"],\n", + " ['Hestia Jones',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Order of the Phoenix'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Neville Longbottom'],\n", + " ['Amycus Carrow', 'common.topic.notable_types', 'Film character'],\n", + " ['Ernie Macmillan',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Slytherin House', 'common.topic.notable_for', 'g.125d1302j'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.02_98hd'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['J. K. Rowling',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Lloyd Alexander'],\n", + " ['Remus Lupin',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['Female', 'common.topic.subjects', 'ZasPorn'],\n", + " ['Oliver Woods',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['The Slug Club',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Ginny Weasley'],\n", + " ['T. H. White', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.09x36bv',\n", + " 'common.webpage.topic',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Draco Malfoy', 'film.film_character.portrayed_in_films', 'm.075nc35'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Jarvey'],\n", + " ['m.0b697x0', 'award.award_honor.award_winner', 'J. K. Rowling'],\n", + " ['Colin Creevey',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['J. K. Rowling',\n", + " 'film.producer.film',\n", + " 'Harry Potter and the Deathly Hallows – Part 2'],\n", + " ['Harry Potter and the Order of the Phoenix', 'film.film.genre', 'Mystery'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Draco Malfoy'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Rubeus Hagrid'],\n", + " ['J. K. Rowling',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'The mind is not a book, to be opened at will and examined at leisure. Thoughts are not etched on the inside of skulls, to be perused by an invader. The mind is a complex and many-layered thing.'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'g.11b6v4ybdd'],\n", + " ['Harry Potter and the Deathly Hallows (Book 7) (Deluxe Edition)',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Lily Evans Potter',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Fiction',\n", + " 'film.film_genre.films_in_this_genre',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Film character', 'type.type.expected_by', 'Killed by'],\n", + " ['Pomona Sprout', 'common.topic.notable_types', 'Film character'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09y2ph2'],\n", + " ['m.09ytphz', 'common.webpage.topic', 'Harry Potter'],\n", + " ['m.09ytphz',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.written_work.author',\n", + " 'J. K. Rowling'],\n", + " ['m.09xjbyv',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.0hhdgp4',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Tom Felton',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'James Potter'],\n", + " ['Professor Horace Slughorn',\n", + " 'fictional_universe.fictional_organization_founder.fictional_organizations_founded',\n", + " 'The Slug Club'],\n", + " ['Eileen Prince',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.09yl2xq',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Professor Minerva McGonagall'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'media_common.quotation_source.quotations',\n", + " 'Indeed, your failure to understand that there are things much worse than death has always been your greatest weakness.'],\n", + " ['Antonin Dolohov',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Pansy Parkinson',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Gwenog Jones'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Norwegian Ridgeback Dragon'],\n", + " ['Charlie Weasley', 'common.topic.notable_types', 'Film character'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " 'Harry Potter Y LA Piedra Filosofal'],\n", + " ['Lucius Malfoy', 'film.film_character.portrayed_in_films', 'm.0jz6g1'],\n", + " ['The mind is not a book, to be opened at will and examined at leisure. Thoughts are not etched on the inside of skulls, to be perused by an invader. The mind is a complex and many-layered thing.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['J. K. Rowling',\n", + " 'film.producer.film',\n", + " 'Harry Potter and the Deathly Hallows - Part I'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Nott Sr.'],\n", + " ['Dudley Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['The Slug Club',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Hermione Granger'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " \"No, I think I'll just go down and have some pudding and wait for it all to turn up.... It always does in the end.\"],\n", + " ['Lord Voldemort',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Slytherin House'],\n", + " ['m.03lwx7f', 'film.performance.character', 'Draco Malfoy'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Justin Finch-Fletchley'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0d5218t'],\n", + " ['m.0gk9sds',\n", + " 'award.award_nomination.nominated_for',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Lucius Malfoy', 'film.film_character.portrayed_in_films', 'm.0csg5rx'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the deathly hallows',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['m.09yd590',\n", + " 'common.webpage.topic',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'g.11b6rb871k'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09wtyly'],\n", + " ['Professor Filius Flitwick',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Gellert Grindelwald',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " 'Elder Wand'],\n", + " ['m.098g827',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter literary series'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " \"Dumbledore's Army\"],\n", + " ['Arthur Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Platform 9 3/4'],\n", + " ['Stephen King', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Slytherin Quidditch Team',\n", + " 'fictional_universe.fictional_organization.parent_organization_in_fiction',\n", + " 'Slytherin House'],\n", + " ['Scotland',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United Kingdom'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Hippocampus'],\n", + " ['J. K. Rowling', 'people.person.profession', 'Author'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Professor Horace Slughorn'],\n", + " ['m.0gxvv0q',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Ronald Weasley', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Fred Weasley'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.09n9nh3'],\n", + " ['Lord Voldemort',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Merope Gaunt'],\n", + " ['J. D. Salinger', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " \"You dare use my own spells against me, Potter? It was I who invented them - I, the Half-Blood Prince! And you'd turn my inventions on me, like your filthy father, would you? I don't think so . . . no.\"],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Corneal abrasion'],\n", + " ['We must try not to sink beneath our anguish, Harry, but battle on.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Professor Severus Snape'],\n", + " ['Ted Lupin',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Remus Lupin'],\n", + " ['George Eliot', 'people.person.profession', 'Novelist'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09ygzbs'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter and the Sorcerer's Stone\"],\n", + " ['Lee Jordan',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.written_work.author',\n", + " 'J. K. Rowling'],\n", + " ['Ronald Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64q8'],\n", + " ['m.02zd4tr',\n", + " 'base.gender.personal_gender_identity.gender_identity',\n", + " 'Male'],\n", + " ['Ancient One',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Molly Weasley'],\n", + " ['Mary GrandPré', 'award.award_winner.awards_won', 'm.0853_v0'],\n", + " ['Time is making fools of us again.',\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Cho Chang'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'g.12105mqq'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter y el Caliz de Fuego'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.120ktb3t'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Percy Weasley',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor House'],\n", + " ['Mrs. Norris',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Walden Macnair',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.editions',\n", + " 'Harry Potter y La Orden del Fenix'],\n", + " ['Gellert Grindelwald',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Durmstrang Institute for Magical Learning'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Deathly Hallows – Part 2'],\n", + " ['James Potter',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['m.09ygz3b',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['G. K. Chesterton', 'influence.influence_node.influenced', 'J. K. Rowling'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.09th3tm'],\n", + " ['Harry Potter and the Goblet of Fire', 'book.book.characters', 'Nott Sr.'],\n", + " ['Ronald Weasley',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Rose Weasley'],\n", + " ['Arthur Weasley',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " \"Never trust anything that can think for itself if you can't see where it keeps its brain.\"],\n", + " ['m.09wtyly', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['Neville Longbottom',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Magic in Harry Potter'],\n", + " ['James Potter',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Prisoner of Azkaban Prepack'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Hermione Granger'],\n", + " ['Harry Potter and the deathly Hallows',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Fred Weasley', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Kidney cancer'],\n", + " ['Draco Malfoy', 'common.topic.webpage', 'm.0bgg_y8'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Professor Albus Dumbledore'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Magic in Harry Potter',\n", + " 'fictional_universe.character_powers.characters_with_this_ability',\n", + " 'Shannon'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Cho Chang'],\n", + " ['m.09n9nh3',\n", + " 'fictional_universe.fictional_employment_tenure.employee',\n", + " 'Poppy Pomfrey'],\n", + " ['Lord Voldemort',\n", + " 'fictional_universe.fictional_character.powers_or_abilities',\n", + " 'Occlumency'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.written_work.author',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Deathly Hallows: Part I'],\n", + " ['m.09xnj6h',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['m.0sgkpw5',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter literary series'],\n", + " ['Draco_Mal.JPG', 'common.image.appears_in_topic_gallery', 'Draco Malfoy'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Susan Bones'],\n", + " ['m.09xwppl',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Shrieking Shack'],\n", + " ['Ernie Macmillan',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Hufflepuff House'],\n", + " ['Harry Potter', 'book.book_subject.works', 'Harry Potter & Imagination'],\n", + " ['Yaxley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['J. K. Rowling', 'book.author.works_written', 'The Casual Vacancy'],\n", + " ['Harry Potter literary series',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0sgp7np'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Professor Sinistra'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09yv55b'],\n", + " ['m.09y14rb', 'common.webpage.topic', 'Death Eaters'],\n", + " ['m.075hqy4',\n", + " 'fictional_universe.marriage_of_fictional_characters.spouses',\n", + " 'Lucius Malfoy'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Number Twelve, Grimmauld Place'],\n", + " ['Harry Potter', 'common.topic.notable_types', 'Film character'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'There is no good or evil: only power and those too weak to seek it.'],\n", + " ['m.09y445w',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " \"I sometimes find, and I am sure you know the feeling, that I simply have too many thoughts and memories crammed into my mind.... At these times... I use the Pensieve. One simply siphons the excess thoughts from one's mind, pours them into the basin, and examines them at one's leisure.\"],\n", + " ['Fear of a name increases fear of the thing itself.',\n", + " 'media_common.quotation.addressee',\n", + " 'Harry Potter'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0sgph7n'],\n", + " ['Ah, music. A magic beyond all we do here!',\n", + " 'media_common.quotation.source',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.11b6tcdj0q'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'common.topic.webpage',\n", + " 'm.09xyf_b'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'common.topic.webpage',\n", + " 'm.09yh025'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'g.11b6v4ybdd'],\n", + " [\"Salazar Slytherin's Locket\",\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Lord Voldemort'],\n", + " ['Jason Cockcroft',\n", + " 'book.illustrator.books_illustrated',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'The Chamber of Secrets'],\n", + " ['Viktor Krum',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Durmstrang Institute for Magical Learning'],\n", + " ['m.09ygm37',\n", + " 'common.webpage.topic',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.written_work.author',\n", + " 'J. K. Rowling'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.0cq71gr'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'g.121lgztv'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'media_common.quotation_source.quotations',\n", + " \"You sort of start thinking anything's possible if you've got enough nerve.\"],\n", + " ['Hermione Granger',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Rose Weasley'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'Wizarding world'],\n", + " ['Viktor Krum',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Poppy Pomfrey',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['heart attack', 'medicine.disease.risk_factors', 'Male'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['J. K. Rowling', 'award.award_nominee.award_nominations', 'm.0b6997_'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'fictional_universe.work_of_fiction.part_of_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['m.075nc35', 'film.performance.character', 'Draco Malfoy'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'People find it far easier to forgive others for being wrong than being right.'],\n", + " ['James Potter',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.09xj12w',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Goblet of Fire', 'book.book.characters', 'Travers'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Urolithiasis'],\n", + " ['Phone Trick',\n", + " 'base.thoroughbredracing.thoroughbred_racehorse.sex',\n", + " 'Male'],\n", + " ['m.09yl33q', 'common.webpage.topic', 'Harry Potter'],\n", + " ['Petunia Evans Dursley',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['House-elf',\n", + " 'fictional_universe.character_species.found_in_fictional_universe',\n", + " 'Harry Potter'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.075sqq9'],\n", + " ['m.0b46q_k', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['m.09xnj6h', 'common.webpage.topic', 'J. K. Rowling'],\n", + " ['J. K. Rowling', 'book.author.works_written', 'g.12mkytxhk'],\n", + " ['Wizards',\n", + " 'book.book_subject.works',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Slytherin Quidditch Team',\n", + " 'fictional_universe.fictional_organization.appears_in_universes',\n", + " 'Harry Potter'],\n", + " ['m.0b495__', 'common.webpage.in_index', 'WSJ Speakeasy Index'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Charlie Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Michael Corner',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Ravenclaw House'],\n", + " ['Fiction',\n", + " 'film.film_genre.films_in_this_genre',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'film.film.sequel',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter Aur Azkaban Ka Qaidi'],\n", + " ['Harry Potter literary series',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0sgph7n'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter y el Prisonero de Azkaban'],\n", + " ['Mrs. Norris',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Nearly Headless Nick', 'common.topic.notable_types', 'Book Character'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'It is our choices...that show what we truly are, far more than our abilities.'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " \"Hufflepuff's cup\"],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Phineas Nigellus Black',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'fictional_universe.work_of_fiction.part_of_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter', 'common.topic.webpage', 'm.09ytphz'],\n", + " ['Fleur Delacour',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['T. H. White', 'people.person.profession', 'Novelist'],\n", + " ['Firenze',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Poppy Pomfrey',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Lucius Malfoy'],\n", + " ['m.0sgllrj', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['Bertha Jorkins', 'common.topic.notable_types', 'Fictional Character'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.0hj5cm7'],\n", + " ['Occlumency', 'common.topic.notable_for', 'g.1259rkg4d'],\n", + " ['Professor Albus Dumbledore',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " [\"Salazar Slytherin's Locket\",\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Dolores Umbridge'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Oliver Woods'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter und der Gefangene von Azkaban'],\n", + " ['Rubeus Hagrid', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['James Potter',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Jason Cockcroft', 'common.topic.notable_types', 'Author'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.11b6swr8r2'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.employers',\n", + " 'm.02xv83k'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Deathly Hallows (Book 7) (Deluxe Edition)'],\n", + " [\"Dumbledore's Army\",\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Fred Weasley'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Susan Bones'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09ym013'],\n", + " ['Horace likes his comfort. He also likes the company of the famous, the successful and the powerful. He enjoys the feeling that he influences these people. He has never wanted to occupy the throne himself. He prefers the backseat; there is more room to spread out.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter and the Goblet of Fire', 'film.film.genre', 'Fantasy'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.written_work.subjects',\n", + " 'Witchcraft'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Fred Weasley'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Merpeople'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Garri Potter i Tainaia Komnata'],\n", + " ['Cardiovascular disease',\n", + " 'people.cause_of_death.includes_causes_of_death',\n", + " 'Myocardial Ischemia'],\n", + " ['Pansy Parkinson',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Sturgis Podmore'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Poppy Pomfrey'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.written_work.subjects',\n", + " 'Schools'],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Avery Sr.'],\n", + " ['Eileen Prince',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Eldred Worple'],\n", + " ['Mary GrandPré', 'award.award_winner.awards_won', 'm.0853_tn'],\n", + " ['Harry Potter and the Deathly Hallows - Part I',\n", + " 'film.film.produced_by',\n", + " 'J. K. Rowling'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Hair loss'],\n", + " ['J. K. Rowling', 'award.award_winner.awards_won', 'm.0b7lcbn'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.0gxvv0q'],\n", + " ['Harry Potter and the Deathly Hallows - Part I',\n", + " 'film.film.genre',\n", + " 'Fiction'],\n", + " ['m.0j8zc4_',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Pomona Sprout'],\n", + " ['J. K. Rowling', 'book.author.works_written', 'El Libro de Las Mascaras'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.written_work.subjects',\n", + " 'Schools'],\n", + " ['Draco Malfoy',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['J. K. Rowling', 'award.ranked_item.appears_in_ranked_lists', 'm.0j0x5cb'],\n", + " ['George Weasley',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Arthur Weasley'],\n", + " ['James Potter',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'book.book_edition.book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Mary GrandPré',\n", + " 'book.illustrator.books_illustrated',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Pomona Sprout',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Oliver Woods',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Fear of a name increases fear of the thing itself.',\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.editions',\n", + " 'Harry Potter y El Caliz de Fuego'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Author'],\n", + " ['m.0kfyrv7',\n", + " 'base.animalnames.animal_specific_name_relationship.gender',\n", + " 'Male'],\n", + " ['Bellatrix Lestrange',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'The Slug Club'],\n", + " ['Lee Jordan',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " 'Harri Potter a Maen yr Athronydd (Welsh Edition)'],\n", + " ['Harry Potter and the Chamber of Secrets', 'book.book.genre', 'Fantasy'],\n", + " ['Lucius Malfoy',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Narcissa Malfoy',\n", + " 'fictional_universe.fictional_character.married_to',\n", + " 'm.075hqy4'],\n", + " ['Harry Potter and the Goblet of Fire', 'film.film.starring', 'm.0jz6g1'],\n", + " ['m.02vhpxt',\n", + " 'fictional_universe.fictional_employment_tenure.employee',\n", + " 'Professor Severus Snape'],\n", + " ['m.0b47kk2',\n", + " 'common.webpage.topic',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Dean Thomas',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Syphilis'],\n", + " ['Lucius Malfoy', 'film.film_character.portrayed_in_films', 'm.0gm70qh'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.0h4z_wh'],\n", + " ['James Potter',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor Quidditch Team'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'g.125_njz84'],\n", + " ['m.075hqxz',\n", + " 'fictional_universe.sibling_relationship_of_fictional_characters.siblings',\n", + " 'Narcissa Malfoy'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter and the Philosopher's Stone (Irish Gaelic Edition)\"],\n", + " ['m.0z1rm_7', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " [\"If you're holding out for universal popularity, I'm afraid you will be in this cabin for a very long time.\",\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['m.099sqv4',\n", + " 'award.award_nomination.nominated_for',\n", + " 'Harry Potter and the Goblet of Fire'],\n", + " ['Tom Felton', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"Parkinson's disease\", 'medicine.disease.risk_factors', 'Male'],\n", + " ['Michael Corner', 'common.topic.notable_types', 'Film character'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'film.film.story_by',\n", + " 'J. K. Rowling'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'fictional_universe.work_of_fiction.setting',\n", + " 'The Burrow'],\n", + " ['Male', 'medicine.risk_factor.diseases', \"Hirschsprung's disease\"],\n", + " ['George Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Walburga Black',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Sirius Black'],\n", + " ['Harry Potter literary series',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0633j1g'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter and the philosopher's stone\"],\n", + " ['Stroke', 'medicine.disease.risk_factors', 'Transient ischemic attack'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Luna Lovegood'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter och den flammande bägaren'],\n", + " ['Draco Malfoy', 'common.topic.webpage', 'm.09yczk5'],\n", + " ['Phineas Nigellus Black',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Multiple myeloma'],\n", + " ['Percy Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.school_in_fiction.students_graduates',\n", + " 'Lord Voldemort'],\n", + " ['m.0hhdgml',\n", + " 'fictional_universe.fictional_employment_tenure.employer',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Jimmy Peakes',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['m.04fkq3c', 'people.place_lived.person', 'J. K. Rowling'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.interior_illustrations_by',\n", + " 'Mary GrandPré'],\n", + " ['Percy Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['m.04fkqfy', 'people.place_lived.person', 'J. K. Rowling'],\n", + " ['Harry Potter and the Deathly Hallows (Book 7) (Library Edition)',\n", + " 'book.book_edition.author_editor',\n", + " 'J. K. Rowling'],\n", + " ['Hermione Granger',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Secretum Secretorum', 'common.topic.notable_types', 'Book'],\n", + " ['Harry Potter and the Deathly Hallows – Part 2',\n", + " 'film.film.genre',\n", + " 'Fantasy'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Imp'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'Happiness can be found, even in the darkest of times, if one only remembers to turn on the light.'],\n", + " ['m.09ytc46',\n", + " 'common.webpage.topic',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Draco Malfoy', 'common.topic.webpage', 'm.0b47kj6'],\n", + " ['Vincent Crabbe',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Crabbe Sr.'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " 'Elder Wand'],\n", + " ['m.09ygwwv',\n", + " 'common.webpage.topic',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Fear of a name increases fear of the thing itself.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Luna Lovegood',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor Quidditch Team'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Myocardial Ischemia'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', \"Re'Em\"],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'The consequences of our actions are so complicated, so diverse, that predicting the future is a very difficult business indeed.'],\n", + " ['Harry Potter', 'common.topic.webpage', 'm.09xwppl'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'media_common.quotation_source.quotations',\n", + " 'It is our choices...that show what we truly are, far more than our abilities.'],\n", + " ['J. K. Rowling', 'book.author.works_written', 'Secretum Secretorum'],\n", + " ['Fred Weasley',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Molly Weasley'],\n", + " ['Professor Albus Dumbledore',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_death',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Harry Potter Fanbase',\n", + " 'freebase.domain_profile.equivalent_topic',\n", + " 'Harry Potter literary series'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'g.11b6v4ybdd'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Doxy'],\n", + " ['m.04fkq5_', 'business.employment_tenure.person', 'J. K. Rowling'],\n", + " ['Dark and difficult times lie ahead. Soon we must all face the choice between what is right and what is easy.',\n", + " 'media_common.quotation.author',\n", + " 'J. K. Rowling'],\n", + " ['Lucius Malfoy',\n", + " 'fictional_universe.fictional_character.has_possessed',\n", + " \"Tom Riddle's diary\"],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Percy Weasley'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Death Eaters', 'common.topic.webpage', 'm.09yhc2y'],\n", + " ['Selwyn',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['Colin Creevey',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " 'Gryffindor House'],\n", + " [\"Marauders' Map\", 'common.topic.notable_types', 'Fictional Object'],\n", + " ['Harry Potter och dödsrelikerna',\n", + " 'book.translation.translation_of',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Deathly Hallows - Part I',\n", + " 'media_common.adaptation.adapted_from',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.translated_work.translations',\n", + " 'Harry Potter och fången från Azkaban'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the prisoner of Azkaban'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_setting.contains',\n", + " 'Forbidden Forest'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.editions',\n", + " 'Harry Potter y La Orden del Fenix'],\n", + " ['English Language',\n", + " 'fictional_universe.fictional_language.where_spoken',\n", + " 'Chudley, England'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Acromantula'],\n", + " ['Harry Potter and the Deathly Hallows', 'book.book.genre', 'Mystery'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'g.125_qqqvc'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'When you have seen as much of life as I have, you will not underestimate the power of obsessive love.'],\n", + " ['People find it far easier to forgive others for being wrong than being right.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Nymphadora Lupin',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Andromeda Tonks'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'Igor Karkaroff'],\n", + " ['Eileen Prince', 'common.topic.notable_types', 'Book Character'],\n", + " ['Nearly Headless Nick',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'common.topic.webpage',\n", + " 'm.09xhhcd'],\n", + " ['m.0sgp67v', 'award.award_nomination.award_nominee', 'J. K. Rowling'],\n", + " ['J. K. Rowling', 'people.person.nationality', 'United Kingdom'],\n", + " ['Molly Weasley',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Ronald Weasley'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter a l'Ecole des Sorciers (French Audio Edition)\"],\n", + " ['Nearly Headless Nick',\n", + " 'fictional_universe.fictional_character.species',\n", + " 'Ghost'],\n", + " [\"We are only as strong as we are united, as weak as we are divided. Lord Voldemort's gift for spreading discord and enmity is very great. We can fight it only by showing an equally strong bond of friendship and trust. Differences of habit and language are nothing at all if our aims are identical and our hearts are open.\",\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " \"Spinner's End\"],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Deathly Hallows – Part 2'],\n", + " ['Colin Creevey',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Arthur Weasley'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.02wm0cg'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Regulus Black'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Sea Serpent'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.040_9t5'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0zm9_69'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Plimpy'],\n", + " ['Order of the Phoenix',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'George Weasley'],\n", + " ['Order of the Phoenix',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Professor Minerva McGonagall'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.organizations',\n", + " 'Slytherin House'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Pancreatic cancer'],\n", + " ['Gryffindor Quidditch Team',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'g.11b6mp5yt4'],\n", + " ['Slytherin House',\n", + " 'base.harrypotter.hogwarts_house.traits',\n", + " 'Resourcefulness'],\n", + " ['Peeves',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Dudley Dursley', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['m.09ytc46',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'media_common.adapted_work.adaptations',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Professor Albus Dumbledore',\n", + " 'base.fictionaluniverse.fictional_murder_victim.killed_by',\n", + " 'Professor Severus Snape'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Hippogriff'],\n", + " ['Professor Severus Snape',\n", + " 'base.fictionaluniverse.fictional_killer.characters_killed',\n", + " 'Professor Albus Dumbledore'],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Professor Severus Snape',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter',\n", + " 'media_common.quotation_addressee.quotations',\n", + " 'The mind is not a book, to be opened at will and examined at leisure. Thoughts are not etched on the inside of skulls, to be perused by an invader. The mind is a complex and many-layered thing.'],\n", + " [\"Tom Riddle's diary\",\n", + " 'fictional_universe.fictional_object.created_by',\n", + " 'Lord Voldemort'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'book.book.characters',\n", + " 'g.11b6v4ybdd'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0x2lgw9'],\n", + " ['Male', 'base.animalnames.animal_gender.animal_names', 'm.05bvjy3'],\n", + " ['House-elf',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Dobby the House Elf'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Lucius Malfoy', 'common.topic.notable_types', 'Film character'],\n", + " ['Cho Chang', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Luna Lovegood', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Dennis Creevey', 'common.topic.notable_types', 'Book Character'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_organization.sub_organization_in_fiction',\n", + " 'Gryffindor House'],\n", + " ['m.09y14rb',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_setting.contains',\n", + " 'Ravenclaw Tower'],\n", + " ['Male', 'biology.hybrid_parent_gender.hybrids', 'm.0bm6pqb'],\n", + " ['J. K. Rowling',\n", + " 'book.author.works_written',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['m.0hhdgp4',\n", + " 'fictional_universe.fictional_employment_tenure.employee',\n", + " 'Neville Longbottom'],\n", + " ['Scotland',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United Kingdom'],\n", + " ['James Potter',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Death Eaters',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Antonin Dolohov'],\n", + " ['Elder Wand',\n", + " 'fictional_universe.fictional_object.owner',\n", + " 'Lord Voldemort'],\n", + " ['Harry Potter literary series',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0sgp67v'],\n", + " [\"Tom Riddle's diary\",\n", + " 'fictional_universe.fictional_object.featured_in_fictional_universe',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Neville Longbottom'],\n", + " ['Lee Jordan', 'common.topic.notable_types', 'Film character'],\n", + " ['Nearly Headless Nick',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.1ym_l5qwv'],\n", + " ['m.09ygwwv',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Male', 'base.gender.gender_identity.people', 'm.04j64rh'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'common.topic.notable_types',\n", + " 'Organization in fiction'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'g.121v85j3'],\n", + " ['J. K. Rowling', 'award.award_winner.awards_won', 'm.0sgph5p'],\n", + " ['Arthur Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'fictional_universe.work_of_fiction.part_of_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Eileen Prince', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Olympe Maxime'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.editions',\n", + " 'Harry Potter and the Prisoner of Azkaban Prepack'],\n", + " ['Bellatrix Lestrange',\n", + " 'fictional_universe.fictional_character.appears_in_these_fictional_universes',\n", + " 'Harry Potter'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.translated_work.translations',\n", + " 'Haris Poteris ir Ugnies Taure'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'g.11b6s_3jgg'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.characters',\n", + " 'Nearly Headless Nick'],\n", + " ['Professor Severus Snape',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ['m.04fkt34', 'education.education.student', 'J. K. Rowling'],\n", + " ['George Weasley',\n", + " 'fictional_universe.fictional_character.organizations',\n", + " \"Dumbledore's Army\"],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Hogwarts staff'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'g.1hb_gfffv'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Andromeda Tonks'],\n", + " ['m.09wtyly',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Lord Voldemort',\n", + " 'base.fictionaluniverse.fictional_killer.characters_killed',\n", + " 'Gellert Grindelwald'],\n", + " ['Professor Severus Snape',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'book.book.characters',\n", + " 'Professor Severus Snape'],\n", + " ['m.0j0x5cb', 'award.ranking.item', 'J. K. Rowling'],\n", + " ['Selwyn', 'common.topic.notable_types', 'Fictional Character'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'media_common.quotation_source.quotations',\n", + " 'Remember, if the time should come when you have to make a choice between what is right, and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort.'],\n", + " ['Firenze',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Remus Lupin',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Ted Lupin'],\n", + " ['m.0j3q_dn', 'film.personal_film_appearance.person', 'J. K. Rowling'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'book.book.characters',\n", + " 'Colin Creevey'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Eileen Prince'],\n", + " ['Happiness can be found, even in the darkest of times, if one only remembers to turn on the light.',\n", + " 'media_common.quotation.source',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Fantasy',\n", + " 'book.book_subject.works',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Redcap'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " 'Harry Potter e la Pietra Filosofale (Italian AudioEdition)'],\n", + " ['m.03lwx7f',\n", + " 'film.performance.film',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " ['Harry Potter', 'fictional_universe.fictional_universe.species', 'Fwooper'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " 'Harry Potter Y LA Piedra Filosofal'],\n", + " ['Ronald Weasley',\n", + " 'fictional_universe.fictional_character.education',\n", + " 'Hogwarts School of Witchcraft and Wizardry'],\n", + " ['Esophageal cancer', 'medicine.disease.risk_factors', 'Male'],\n", + " ['Harry Potter and the Goblet of Fire',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Fantasy'],\n", + " ['Fiction', 'media_common.media_genre.child_genres', 'Mystery'],\n", + " ['It is true, however, that those who have mastered Legilimency are able, under certain conditions, to delve into the minds of their victims and to interpret their findings correctly....',\n", + " 'media_common.quotation.subjects',\n", + " 'Lord Voldemort'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.species',\n", + " 'Streeler'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'common.topic.article',\n", + " 'm.015plz'],\n", + " ['Male', 'medicine.risk_factor.diseases', 'Esophageal cancer'],\n", + " ['Harry Potter and the Half-Blood Prince',\n", + " 'common.topic.notable_types',\n", + " 'Book'],\n", + " ['Gryffindor House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Colin Creevey'],\n", + " ['m.09w7pvg',\n", + " 'common.webpage.in_index',\n", + " 'Entertainment Weekly annotation index'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'freebase.valuenotation.has_value',\n", + " 'Date written'],\n", + " ['Professor Filius Flitwick',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Horace likes his comfort. He also likes the company of the famous, the successful and the powerful. He enjoys the feeling that he influences these people. He has never wanted to occupy the throne himself. He prefers the backseat; there is more room to spread out.',\n", + " 'media_common.quotation.subjects',\n", + " 'Professor Horace Slughorn'],\n", + " [\"Dumbledore's Army\",\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'Colin Creevey'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'Dark and difficult times lie ahead. Soon we must all face the choice between what is right and what is easy.'],\n", + " ['Remus Lupin',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['J. K. Rowling',\n", + " 'film.film_story_contributor.film_story_credits',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " \"Hagrid's Hut\"],\n", + " ['Dudley Dursley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Harry Potter', 'internet.website_category.sites', 'The Leaky Cauldron'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.fictional_objects',\n", + " 'Elder Wand'],\n", + " ['The Bloody Baron',\n", + " 'book.book_character.appears_in_book',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " ['J. K. Rowling',\n", + " 'book.author.book_editions_published',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Charlie Weasley'],\n", + " ['Emily Brontë', 'people.person.nationality', 'England'],\n", + " ['Harry Potter and the Goblet of Fire', 'book.book.characters', 'Wilkes'],\n", + " ['Rubeus Hagrid',\n", + " 'fictional_universe.fictional_character.employers',\n", + " 'm.0j8zcvf'],\n", + " ['Charlie Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Half-Blood Prince'],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.translated_work.translations',\n", + " 'Hareios Poter Kai he tou Philosophou Lithos'],\n", + " ['Petunia Evans Dursley',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Female'],\n", + " ['Slytherin House',\n", + " 'fictional_universe.fictional_organization.members',\n", + " 'The Bloody Baron'],\n", + " ['Harry Potter', 'common.topic.image', 'harry-potter-books.jpg'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'Fear of a name increases fear of the thing itself.'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Durmstrang Institute for Magical Learning'],\n", + " ['Harry Potter',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'fictional_universe.fictional_employer.employees',\n", + " 'm.02xv83k'],\n", + " ['Hogwarts School of Witchcraft and Wizardry',\n", + " 'common.topic.webpage',\n", + " 'm.09xwpny'],\n", + " ['Harry Potter and the Prisoner of Azkaban',\n", + " 'common.topic.notable_types',\n", + " 'Book'],\n", + " ['Harry Potter and the Chamber of Secrets',\n", + " 'book.book.editions',\n", + " 'Harry Potter und die Kammer des Schrekens'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Little Whinging'],\n", + " ['Igor Karkaroff', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Professor Albus Dumbledore',\n", + " 'fictional_universe.fictional_character.quotations',\n", + " 'Youth cannot know how age thinks and feels. But old men are guilty if they forget what it was to be young.'],\n", + " ['Witchcraft',\n", + " 'film.film_subject.films',\n", + " 'Harry Potter and the Chamber of Secrets'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'book.book.characters',\n", + " 'Pomona Sprout'],\n", + " ['Harry Potter',\n", + " 'fictional_universe.fictional_universe.works_set_here',\n", + " 'Harry Potter and the Order of the Phoenix'],\n", + " ['Fiction',\n", + " 'film.film_genre.films_in_this_genre',\n", + " \"Harry Potter and the Philosopher's Stone\"],\n", + " [\"Harry Potter and the Philosopher's Stone\",\n", + " 'book.book.editions',\n", + " \"Harry Potter and the Philosopher's Stone (Harry Potter)\"],\n", + " ['Gellert Grindelwald',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Deathly Hallows'],\n", + " ['George Weasley',\n", + " 'book.book_character.appears_in_book',\n", + " 'Harry Potter and the Prisoner of Azkaban'],\n", + " ['Harry Potter and the Order of the Phoenix',\n", + " 'media_common.quotation_source.quotations',\n", + " \"Kreacher is what he has been made by wizards, Harry. Yes, he is to be pitied. His existence has been as miserable as your friend Dobby's. He was forced to do Sirius's bidding, because Sirius was the last of the family to which he was enslaved, but he felt no true loyalty to him. And whatever Kreacher's faults, it must be admitted that Sirius did nothing to make Kreacher's lot easier.\"],\n", + " ['Ancient One', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Curiosity is not a sin.... But we should exercise caution with our curiosity... yes, indeed.',\n", + " 'media_common.quotation.spoken_by_character',\n", + " 'Professor Albus Dumbledore'],\n", + " ['J. K. Rowling', 'common.topic.webpage', 'm.09xj12w'],\n", + " ['Harry Potter', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Harry Potter and the Deathly Hallows',\n", + " 'common.topic.webpage',\n", + " 'm.0b497nh'],\n", + " ['J. K. Rowling',\n", + " 'people.person.quotations',\n", + " 'Remember, if the time should come when you have to make a choice between what is right, and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort.'],\n", + " ['Tom Felton', 'film.actor.film', 'm.075nc3k'],\n", + " [\"I sometimes find, and I am sure you know the feeling, that I simply have too many thoughts and memories crammed into my mind.... At these times... I use the Pensieve. One simply siphons the excess thoughts from one's mind, pours them into the basin, and examines them at one's leisure.\",\n", + " 'media_common.quotation.addressee',\n", + " 'Harry Potter'],\n", + " ['Poppy Pomfrey',\n", + " 'fictional_universe.fictional_character.character_created_by',\n", + " 'J. K. Rowling'],\n", + " ...],\n", + " [['United States of America',\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'g.125_r6z32'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Willamette Week'],\n", + " ['Virtua Tennis 4', 'cvg.game_version.publisher', 'Sega'],\n", + " ['m.0wg942s',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['Saint Croix, U.S. Virgin Islands',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Time zone(s)'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3_4mj'],\n", + " ['Mount Cook', 'location.location.containedby', 'Saint Elias Mountains'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Isabel'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.1245_blph'],\n", + " ['United Kingdom',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Economist'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'New Jersey'],\n", + " ['Mexican–American War',\n", + " 'base.culturalevent.event.entity_involved',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc3x3xf'],\n", + " ['Virginia', 'location.location.containedby', 'Contiguous United States'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'North Carolina'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Gazette van Detroit'],\n", + " ['Missouri',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['m.0478pmb', 'military.military_command.military_conflict', 'Vietnam War'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3d37h'],\n", + " ['United States of America',\n", + " 'location.statistical_region.net_migration',\n", + " 'g.1s064935v'],\n", + " ['m.03fx85y',\n", + " 'government.government_position_held.office_position_or_title',\n", + " 'President of the United States'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " \"United States men's national soccer team\"],\n", + " ['Canada', 'location.location.partially_contains', 'Richelieu River'],\n", + " ['Lego Pirates of the Caribbean: The Video Game',\n", + " 'cvg.computer_videogame.versions',\n", + " 'LEGO Pirates of the Caribbean: The Video Game'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3_svg'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc48__w'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'White River'],\n", + " ['France',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['m.0b658s_',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['m.03fx83v',\n", + " 'government.government_position_held.district_represented',\n", + " 'United States of America'],\n", + " ['United States, with Territories',\n", + " 'location.country.first_level_divisions',\n", + " 'Northern Mariana Islands'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'm.0nfsk3v'],\n", + " ['Canada', 'base.locations.countries.continent', 'North America'],\n", + " ['Simon Marsden',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Joseph Hollister'],\n", + " ['United States of America',\n", + " 'location.statistical_region.religions',\n", + " 'm.03q1lwt'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124617f8p'],\n", + " ['Saint Croix, U.S. Virgin Islands',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3v297'],\n", + " ['United States of America',\n", + " 'location.location.time_zones',\n", + " 'Pacific Time Zone'],\n", + " ['m.0b652wd',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2007 World Championships in Athletics'],\n", + " ['Tropical Storm Arthur',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'United States of America'],\n", + " ['Albert Stroller', 'common.topic.notable_types', 'TV Character'],\n", + " ['United States of America',\n", + " 'location.statistical_region.renewable_freshwater_per_capita',\n", + " 'g.1hhc3pmsf'],\n", + " ['Virtua Tennis 4', 'common.topic.notable_types', 'Video Game Version'],\n", + " [\"United States men's national basketball team\",\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['New York', 'location.location.containedby', 'Contiguous United States'],\n", + " ['Navassa Island', 'location.location.containedby', 'Americas'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'LEGO Pirates of the Caribbean: The Video Game'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Liberator'],\n", + " ['Sarah Andersen',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc3zxmv'],\n", + " ['Greg Andersen',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['Father',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['Nebraska', 'common.topic.notable_types', 'US State'],\n", + " ['Cowlitz County', 'location.location.time_zones', 'Pacific Time Zone'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'g.1245_zszh'],\n", + " ['Contiguous United States', 'location.location.contains', 'North Dakota'],\n", + " ['m.0b659jm',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['Cuban Missile Crisis',\n", + " 'base.culturalevent.event.entity_involved',\n", + " 'North Atlantic Treaty Organization (NATO)'],\n", + " ['North Carolina',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Father', 'common.topic.notable_types', 'Film character'],\n", + " ['Location', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['United States House of Representatives',\n", + " 'government.governmental_body.offices_positions',\n", + " 'United States Representative'],\n", + " ['New Zealand', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['US vs. USSR', 'base.rivalries.rivalry.rival', 'United States of America'],\n", + " ['Harry S. Truman', 'people.person.nationality', 'United States of America'],\n", + " ['m.04jx14r',\n", + " 'military.military_command.military_commander',\n", + " 'Harry S. Truman'],\n", + " ['Oak Hill',\n", + " 'base.locations.place_in_the_world.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Washington'],\n", + " ['m.0gprnjn', 'military.casualties.type_of_casualties', 'Military - Dead'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2000 Summer Olympics'],\n", + " ['United States of America',\n", + " 'location.location.time_zones',\n", + " 'Mountain Time Zone'],\n", + " ['m.0wg93wf',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc4b1w3'],\n", + " ['Rockfish Township, Cumberland County, North Carolina',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'South Dakota'],\n", + " ['Contiguous United States', 'location.location.contains', 'Michigan'],\n", + " ['Olivia Benson', 'fictional_universe.fictional_character.gender', 'Female'],\n", + " ['Saint Elias Mountains',\n", + " 'geography.mountain_range.mountains',\n", + " 'Mount Hubbard'],\n", + " ['Dota 2', 'cvg.game_version.developer', 'Valve Corporation'],\n", + " ['Don Carlos Buell',\n", + " 'military.military_commander.military_commands',\n", + " 'm.049y2wj'],\n", + " ['Malden Evening News',\n", + " 'book.newspaper.circulation_areas',\n", + " 'United States of America'],\n", + " ['Japan',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Sovereign state'],\n", + " ['Fortymile River', 'location.location.containedby', 'North America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4j__2'],\n", + " ['Space Race', 'time.event.locations', 'United States of America'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Utah'],\n", + " ['m.04kc46r',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['Video Game Version',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'New Jersey'],\n", + " ['Utah',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['m.0112vj06',\n", + " 'sports.competitor_country_relationship.competitor',\n", + " 'Serena Williams'],\n", + " ['m.0gps9h3',\n", + " 'military.casualties.military_conflict',\n", + " 'Attack on Pearl Harbor'],\n", + " ['West Virginia', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['m.049375v',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['m.0wg942y',\n", + " 'location.partial_containment_relationship.partially_contains',\n", + " 'Red River of the North'],\n", + " ['Lojban',\n", + " 'language.human_language.countries_spoken_in',\n", + " 'United States of America'],\n", + " ['m.063gzhv',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['Contiguous United States', 'location.location.contains', 'Arkansas'],\n", + " ['The Chaos Engine 2',\n", + " 'cvg.computer_videogame.publisher',\n", + " 'United States of America'],\n", + " ['Louisiana',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['United States national rugby union team',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['North Atlantic Treaty Organization (NATO)',\n", + " 'organization.organization.founders',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nfsbhl'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfry1z'],\n", + " ['France',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1900 Summer Olympics'],\n", + " ['Union Jack', 'book.periodical.language', 'English Language'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.subsumes',\n", + " 'United States of America'],\n", + " ['m.0hnzf2l',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.0493797'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Maine'],\n", + " ['Josh Kirby, Time Warrior: Planet of the Dino Knights',\n", + " 'film.film.language',\n", + " 'English Language'],\n", + " ['US Federal District is not suited to be the administrative division type for U.S. Federal District.',\n", + " 'base.uncommon.exception.involves',\n", + " 'U.S. federal district'],\n", + " ['Maine', 'location.location.partially_contains', 'Caribou Mountain'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wg93wf'],\n", + " ['Canada', 'location.location.partially_contains', 'Kettle River'],\n", + " ['Canada', 'location.location.partially_contains', 'Skagit River'],\n", + " ['Canada', 'location.location.containedby', 'North America'],\n", + " ['New York City', 'base.biblioness.bibs_topic.subsumes', 'New York City'],\n", + " ['Drew Lou Who',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Lou Lou Who'],\n", + " ['Palmyra Atoll', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['Buena Vista city', 'location.location.containedby', 'Virginia'],\n", + " ['m.0zdgbqg',\n", + " 'sports.competitor_country_relationship.competitor',\n", + " \"United States men's national basketball team\"],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b64vmj'],\n", + " ['Manhattan', 'location.location.containedby', 'New York City'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.0493792'],\n", + " ['United States, with Territories',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'United States of America'],\n", + " ['Capitalism: A Love Story',\n", + " 'film.film.subjects',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'American Free Press'],\n", + " ['m.0hnzf6_',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfrxm2'],\n", + " ['m.0wg942g',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['m.0b6599q',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Gold medal'],\n", + " ['m.0b653p5',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Gold medal'],\n", + " ['United States of America',\n", + " 'sports.sport_country.multi_event_tournaments_participated_in',\n", + " '2012 World Mountain Running Championships'],\n", + " ['Joseph Hollister',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Male'],\n", + " ['United States of America',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc5311h'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.0493748'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'g.1245yxht1'],\n", + " ['Marshall Islands',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['St. Clair River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['Newspaper', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Taiwan',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Sovereign state'],\n", + " ['United States Secretary of Housing and Urban Development',\n", + " 'common.topic.notable_types',\n", + " 'Government Office or Title'],\n", + " ['m.063gzk2',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Endangered'],\n", + " ['Nevada',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.1245ywjjb'],\n", + " ['United States of America',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc54vgh'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3l9c6'],\n", + " ['m.063gzl2',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['Serena Benson',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Olivia Benson'],\n", + " ['m.0hnzf4p',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['United States of America',\n", + " 'base.litcentral.focal_location.priority_species',\n", + " 'm.063gzlb'],\n", + " ['Arizona', 'freebase.valuenotation.is_reviewed', 'Minimum wage'],\n", + " ['Korean War', 'military.military_conflict.casualties', 'm.043wpm7'],\n", + " ['New Hampshire',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Missouri',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['m.049374l',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['Arizona', 'location.location.partially_contains', 'Santa Cruz River'],\n", + " ['Northern Mariana Islands', 'common.topic.notable_types', 'Country'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.04jx303'],\n", + " ['United States of America',\n", + " 'government.political_district.representatives',\n", + " 'm.03fx7hj'],\n", + " ['Missouri', 'location.location.time_zones', 'Central Time Zone'],\n", + " ['Lego Pirates of the Caribbean: The Video Game',\n", + " 'common.topic.notable_types',\n", + " 'Video game'],\n", + " ['Sovereign state',\n", + " 'base.aareas.schema.administrative_area_type.pertains_to',\n", + " 'Earth'],\n", + " ['Arkansas',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'film.film_location.featured_in_films',\n", + " 'Have Tig at Your Party'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.game',\n", + " 'Lego Pirates of the Caribbean: The Video Game'],\n", + " ['Alaska', 'location.location.containedby', 'United States of America'],\n", + " ['Navassa Island',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States Minor Outlying Islands'],\n", + " ['France',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1936 Summer Olympics'],\n", + " ['Jonah Malcolm',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['La Opinión',\n", + " 'book.newspaper.circulation_areas',\n", + " 'United States of America'],\n", + " ['Alsek River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['m.02kljy2',\n", + " 'military.force_strength.combatant',\n", + " 'United States of America'],\n", + " ['North Star', 'common.topic.notable_types', 'Newspaper'],\n", + " ['United States of America',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc4ygxw'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfrylf'],\n", + " ['m.0b64y5x',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2009 World Championships in Athletics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3nx19'],\n", + " ['Similkameen River',\n", + " 'location.location.partially_containedby',\n", + " 'Washington'],\n", + " ['Drew Lou Who',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['m.02hvygx',\n", + " 'aviation.aircraft_ownership_count.aircraft_owner',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Mount Saint Elias'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Dziennik Związkowy'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs2z0'],\n", + " ['Sonic Adventure', 'cvg.game_version.regions', 'Japan'],\n", + " ['m.0hnzf3s',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['Battle of Iwo Jima', 'military.military_conflict.commanders', 'm.0gpry74'],\n", + " ['Endangered',\n", + " 'base.conservationaction.priority_level.priorities_at_this_level',\n", + " 'm.063gzhv'],\n", + " ['Good Neighbor Peak',\n", + " 'location.location.containedby',\n", + " 'Saint Elias Mountains'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4zqz7'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Vermont'],\n", + " ['m.063gzjg',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['m.0b653rz',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2007 World Championships in Athletics'],\n", + " ['North America', 'location.location.containedby', 'Americas'],\n", + " ['Contiguous United States',\n", + " 'location.location.time_zones',\n", + " 'Pacific Time Zone'],\n", + " ['Alabama',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['m.0b659v5',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Bronze medal'],\n", + " ['Nevada',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['m.0hnzfc8',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['Endangered',\n", + " 'base.conservationaction.priority_level.priorities_at_this_level',\n", + " 'm.063gzlg'],\n", + " ['m.0nfsrq0',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['m.0nfsp__',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.religions',\n", + " 'm.03q1lvq'],\n", + " ['Pacific Ocean', 'location.location.adjoin_s', 'm.02wtp8v'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'g.11b60_8p9h'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc3qkrl'],\n", + " ['New York City',\n", + " 'media_common.quotation_subject.quotations_about_this_subject',\n", + " '\\\\\"History dressed up in the glow of love’s kiss turned grief into beauty.\\\\\"'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.regions',\n", + " 'United States of America'],\n", + " ['m.0nfshmz',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Military expenditure as percentage of GDP, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United Nations', 'organization.organization.founders', 'United Kingdom'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.049y2vz'],\n", + " ['Georgia',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['Zachary Taylor',\n", + " 'military.military_commander.military_commands',\n", + " 'm.04h_h41'],\n", + " ['Mexico',\n", + " 'base.mystery.cryptid_area_of_occurrence.cryptid_s_found_here',\n", + " 'Chupacabra'],\n", + " ['Oklahoma', 'location.location.containedby', 'United States of America'],\n", + " ['m.0b64zy9',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Bronze medal'],\n", + " ['Mexico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2014 Winter Olympics'],\n", + " ['m.0nfsbgb',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'GNI per capita in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Mount Augusta', 'location.location.containedby', 'Pacific Coast Ranges'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Indiana'],\n", + " ['2007 World Championships in Athletics',\n", + " 'sports.multi_event_tournament.athletic_performances',\n", + " 'm.0b652tp'],\n", + " ['United States of America',\n", + " 'location.statistical_region.diesel_price_liter',\n", + " 'g.12cp_kgjb'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Village Voice'],\n", + " ['New Zealand',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2014 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.poverty_rate_2dollars_per_day',\n", + " 'g.11b6cy8pxl'],\n", + " ['m.0nfspzr',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Arkansas',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['Japan',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'Sonic Adventure'],\n", + " ['TV Character', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Alaska', 'common.topic.notable_types', 'US State'],\n", + " ['Australia', 'periodicals.newspaper_circulation_area.newspapers', 'Zaman'],\n", + " ['U.S. federal district', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['United States Minor Outlying Islands',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'The Adventures of Rad Gravity'],\n", + " ['m.0nfsrq8',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b64z72'],\n", + " ['Counter-Strike: Global Offensive',\n", + " 'cvg.game_version.publisher',\n", + " 'Valve Corporation'],\n", + " ['Contiguous United States', 'location.location.contains', 'Pennsylvania'],\n", + " ['Manhattan',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The New York Times'],\n", + " ['m.066nmg1',\n", + " 'base.athletics.athletics_championships_competition_athlete_relationship.athlete_s',\n", + " 'Tyson Gay'],\n", + " ['United Kingdom',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1984 Summer Olympics'],\n", + " ['m.0wg941m',\n", + " 'location.partial_containment_relationship.partially_contains',\n", + " 'Pend Oreille River'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzfb0'],\n", + " ['United States of America', 'common.topic.notable_for', 'g.1255r20f5'],\n", + " ['m.0nfsk7w',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc44sbs'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Colorado River'],\n", + " ['Josh Kirby, Time Warrior: Planet of the Dino Knights',\n", + " 'film.film.country',\n", + " 'United States of America'],\n", + " ['m.04ml3kt', 'military.force_strength.combatant', 'Australia'],\n", + " ['United States of America',\n", + " 'cvg.cvg_publisher.games_published',\n", + " 'Point Blank'],\n", + " ['River', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Greg Andersen',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Natasha Andersen'],\n", + " ['South Korea', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Navassa Island',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.datedlocationtest.dated_location_test.joined_by',\n", + " 'm.0b9td6g'],\n", + " ['United States of America',\n", + " 'base.litcentral.focal_location.priority_species',\n", + " 'm.063gzhq'],\n", + " ['Oregon', 'location.location.containedby', 'United States of America'],\n", + " ['m.04h_gf3',\n", + " 'military.military_command.military_conflict',\n", + " 'Mexican–American War'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'g.1hhc3zxqd'],\n", + " ['1992 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzfc8'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245z2kxy'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245_nwwx'],\n", + " ['Massachusetts',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Arkansas', 'freebase.valuenotation.is_reviewed', 'Minimum wage'],\n", + " ['2002 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'New Zealand'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'm.0nfshln'],\n", + " ['Hawaii', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['Lego Pirates of the Caribbean: The Video Game',\n", + " 'cvg.computer_videogame.publisher',\n", + " 'Disney Interactive Studios'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'common.topic.notable_types',\n", + " 'Video Game Version'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc3bykr'],\n", + " ['United States of America',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc52ms7'],\n", + " ['United States of America',\n", + " 'military.military_combatant.casualties',\n", + " 'm.0gps9gs'],\n", + " ['Northern Mariana Islands',\n", + " 'location.country.languages_spoken',\n", + " 'Spanish Language'],\n", + " ['Oregon',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.litcentral.focal_location.priority_species',\n", + " 'm.063gzkq'],\n", + " ['Australia',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Alabama'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs39w'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " 'Team RadioShack'],\n", + " ['m.048_4q3',\n", + " 'location.imports_and_exports.exported_to',\n", + " 'United States of America'],\n", + " ['Serena Williams',\n", + " 'sports.tournament_event_competitor.events_competed_in',\n", + " 'm.0wzp362'],\n", + " ['Mexico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2002 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'm.0nfspz7'],\n", + " ['South Dakota', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['Australia',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zx1k_'],\n", + " ['United States of America',\n", + " 'organization.organization_member.member_of',\n", + " 'm.04dj48z'],\n", + " ['Morgen Freiheit', 'book.newspaper.circulation_areas', 'New York City'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'g.12460tjy5'],\n", + " ['Similkameen River',\n", + " 'location.location.partially_contained_by',\n", + " 'm.0wg93_w'],\n", + " ['m.0b6562w',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2007 World Championships in Athletics'],\n", + " ['The London 2012 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'South Korea'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0svns44'],\n", + " ['Contiguous United States',\n", + " 'location.location.time_zones',\n", + " 'Mountain Time Zone'],\n", + " ['Canada', 'location.location.partially_contains', 'Caribou Mountain'],\n", + " ['Cindy Lou Who',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Lou Lou Who'],\n", + " ['Skagit River', 'location.location.partially_containedby', 'Washington'],\n", + " ['New York City',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Morgen Freiheit'],\n", + " ['m.0pl31jm',\n", + " 'military.military_command.military_commander',\n", + " 'Zachary Taylor'],\n", + " ['Japan',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['m.0wg93zc',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['United States Minor Outlying Islands',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States, with Territories'],\n", + " ['World Trade Organization', 'organization.organization.founders', 'Canada'],\n", + " ['United States National Economic Council',\n", + " 'government.governmental_body.jurisdiction',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.04fxlgq'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wg940n'],\n", + " ['Colorado River', 'location.location.partially_containedby', 'Colorado'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Minnesota'],\n", + " ['Mount Quincy Adams',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['m.04fvgm0',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['Guile', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Marshall Islands',\n", + " 'location.country.official_language',\n", + " 'English Language'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc4nfn_'],\n", + " ['m.052w0ky',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['United States Equestrian Team',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['m.049377h', 'location.imports_and_exports.imported_from', 'Canada'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc51plr'],\n", + " ['m.0b7bysn',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['Tennis',\n", + " 'olympics.olympic_sport.olympic_games_contested',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['U.S. state', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Virtua Tennis 4', 'cvg.game_version.developer', 'Sega'],\n", + " ['Hurricane Lidia', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Maryland', 'freebase.valuenotation.is_reviewed', 'Minimum wage'],\n", + " ['m.0nfs3cf',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Halls Stream', 'location.location.partially_containedby', 'New Hampshire'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['Canada', 'location.location.partially_contains', 'Great Lakes'],\n", + " ['m.0gkhcxv',\n", + " 'base.sharing.sharing_relationship.locations',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " \"Investor's Business Daily\"],\n", + " ['Abby Bradford',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Mr Bradford'],\n", + " ['m.0wg93zp',\n", + " 'location.partial_containment_relationship.partially_contains',\n", + " 'Okanogan River'],\n", + " ['Massachusetts',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Barnstable County', 'location.location.containedby', 'Massachusetts'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Barnstable County'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc388q4'],\n", + " ['m.063gzjz',\n", + " 'base.conservationaction.documented_priority_species.source',\n", + " 'Birds of Conservation Concern 2008'],\n", + " ['Kettle River', 'geography.river.basin_countries', 'Canada'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4lgkd'],\n", + " ['m.0hnzf0v',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['1976 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc3sn2v'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.124611vf4'],\n", + " ['North Dakota',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc47bdp'],\n", + " ['m.0nfrxzf',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf6q'],\n", + " ['United States of America',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc3fgyq'],\n", + " ['Keith Goodman',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['m.063gzlg',\n", + " 'base.conservationaction.documented_priority_species.source',\n", + " 'Birds of Conservation Concern 2008'],\n", + " ['Hawaii',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['United States Fed Cup team',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['New York',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'New York City'],\n", + " ['m.06dn7lr',\n", + " 'base.athletics.athletics_championships_competition_athlete_relationship.championships',\n", + " '2007 World Championships in Athletics'],\n", + " ['Canada',\n", + " 'base.charities.geographic_scope.charities',\n", + " \"Starlight Children's Foundation\"],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245zsmrz'],\n", + " ['United Kingdom',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'Sovereign state'],\n", + " ['United States of America',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'Federal judiciary of the United States'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.publisher',\n", + " 'Disney Interactive Studios'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Arlington County'],\n", + " ['United States of America',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc3v5w2'],\n", + " ['United States of America',\n", + " 'location.statistical_region.brain_drain_percent',\n", + " 'g.1hhc51ltb'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.12tb6fdqm'],\n", + " ['m.0b652rz',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2007 World Championships in Athletics'],\n", + " ['m.04fvhd3',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['The London 2012 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['m.0wjpmrn',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.048z_rr'],\n", + " ['Boulder Dash', 'cvg.game_version.regions', 'United States of America'],\n", + " ['m.0b652tp',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2007 World Championships in Athletics'],\n", + " ['Canada', 'location.statistical_region.places_exported_to', 'm.049377h'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.049375p'],\n", + " ['United Kingdom',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Hunchak'],\n", + " ['Yukon River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4j8mm'],\n", + " ['American Free Press', 'book.periodical.language', 'English Language'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf5t'],\n", + " ['Tropical Storm Fay',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Tennessee'],\n", + " ['m.096hjmy',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Bronze medal'],\n", + " ['Vietnam War', 'military.military_conflict.force_strengths', 'm.02h782s'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'LEGO Pirates of the Caribbean: The Video Game'],\n", + " ['Canada', 'location.location.time_zones', 'Mountain Time Zone'],\n", + " ['Missouri',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1246058s8'],\n", + " ['Japan',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Stars and Stripes'],\n", + " ['United States of America',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc4_k1s'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.1246120tm'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245z0xf5'],\n", + " ['Cuban Missile Crisis',\n", + " 'base.culturalevent.event.entity_involved',\n", + " 'United Kingdom'],\n", + " ['Endangered and Threatened Wildlife and Plants; Review of Native Species That Are Candidates for Listing as Endangered or Threatened; Annual Notice of Findings on Resubmitted Petitions; Annual Description of Progress on Listing Actions',\n", + " 'base.conservationaction.priority_source.priority_taxa',\n", + " 'm.052w0k9'],\n", + " ['Battle of Okinawa',\n", + " 'base.culturalevent.event.entity_involved',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs3b_'],\n", + " ['m.0nfsb95',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'm.0nfspyr'],\n", + " ['Nevada', 'location.location.partially_contains', 'Colorado River'],\n", + " ['m.0b7rn4z',\n", + " 'base.datedlocationtest.dated_location_succession.succeeded_by',\n", + " 'United States of America'],\n", + " ['Willamette Week',\n", + " 'book.newspaper.circulation_areas',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'Boulder Dash'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'g.1245yvl9s'],\n", + " ['m.0nfrxxn',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Contiguous United States', 'location.location.contains', 'Maryland'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf62'],\n", + " ['White River', 'location.location.containedby', 'North America'],\n", + " ['Government Office or Title',\n", + " 'freebase.type_hints.included_types',\n", + " 'Topic'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " \"People's World\"],\n", + " ['Aroostook River', 'geography.river.mouth', 'Saint John River'],\n", + " ['m.0b64vmj',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2009 World Championships in Athletics'],\n", + " ['m.0nfrxm2',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzfcv'],\n", + " ['Transatlantic Business Dialogue',\n", + " 'common.topic.notable_types',\n", + " 'Organization'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc50pl6'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1s066rvwd'],\n", + " ['Lego Pirates of the Caribbean: The Video Game',\n", + " 'cvg.computer_videogame.versions',\n", + " 'LEGO Pirates of the Caribbean: The Video Game'],\n", + " ['Canada', 'location.location.partially_contains', 'Fortymile River'],\n", + " ['m.0wg9414',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['Canada', 'location.location.partially_contains', 'Aroostook River'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc385bv'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.11b71v6t4q'],\n", + " ['m.02wtp98',\n", + " 'location.adjoining_relationship.adjoins',\n", + " 'United States of America'],\n", + " ['m.0gprtxt',\n", + " 'military.force_strength.combatant',\n", + " 'United States of America'],\n", + " ['Boundary Peak',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.1245z4gk5'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'Point Blank DS'],\n", + " ['m.0nfsk93',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Elizabeth Wiatt', 'common.topic.notable_types', 'Film character'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.developer',\n", + " \"Traveller's Tales\"],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc53ph2'],\n", + " ['United States of America', 'location.country.internet_tld', 'us'],\n", + " ['m.010h8r53',\n", + " 'military.force_strength.combatant',\n", + " 'United States of America'],\n", + " ['Astonishia Story', 'cvg.game_version.regions', 'United States of America'],\n", + " ['Hurricane Isis', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['Contiguous United States', 'location.location.contains', 'Idaho'],\n", + " ['Guam', 'common.topic.notable_types', 'Country'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'g.1245zt6g_'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Maine'],\n", + " ['Canada', 'location.location.partially_contains', 'Pend Oreille River'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4_7l_'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['m.0b653rz',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc4y46z'],\n", + " ['Alaska', 'location.location.partially_contains', 'Mount Quincy Adams'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Washington, D.C.'],\n", + " ['Wake Island',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['Star Wars: Republic Commando',\n", + " 'base.wikipedia_infobox.video_game.platforms',\n", + " 'Microsoft Windows'],\n", + " ['m.04fvkjd',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['m.0b6510s',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2009 World Championships in Athletics'],\n", + " ['Alaska', 'location.location.partially_contains', 'Mount Alverstone'],\n", + " ['Porcupine River', 'geography.river.basin_countries', 'Canada'],\n", + " ['Manhattan',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States of America'],\n", + " ['Viceroy Miami',\n", + " 'location.location.primarily_containedby',\n", + " 'United States of America'],\n", + " ['Colorado River', 'location.location.partially_containedby', 'Utah'],\n", + " ['Mexican–American War',\n", + " 'military.military_conflict.commanders',\n", + " 'm.04h_gf3'],\n", + " ['The Guardian Weekly',\n", + " 'book.newspaper.circulation_areas',\n", + " 'United Kingdom'],\n", + " ['m.0wp5grg',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['War of 1812', 'time.event.locations', 'North America'],\n", + " ['United States of America',\n", + " 'military.military_combatant.force_deployments',\n", + " 'm.04ml3kt'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " 'United States Fed Cup team'],\n", + " ['Saint Francis River', 'location.location.partially_containedby', 'Maine'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " 'United States Equestrian Team'],\n", + " ['Palmyra Atoll',\n", + " 'location.location.containedby',\n", + " 'United States Minor Outlying Islands'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Saint John'],\n", + " ['m.0nfsbgl',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc47nxz'],\n", + " ['Hurricane Paul', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['m.052w0k1',\n", + " 'base.conservationaction.documented_priority_species.source',\n", + " 'Endangered and Threatened Wildlife and Plants; Review of Native Species That Are Candidates for Listing as Endangered or Threatened; Annual Notice of Findings on Resubmitted Petitions; Annual Description of Progress on Listing Actions'],\n", + " ['Albert Stroller',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Kathleen'],\n", + " ['Utah',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'fictional_universe.fictional_setting.fictional_characters_born_here',\n", + " 'Kathleen'],\n", + " ['United States of America',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc4wgy_'],\n", + " ['Contiguous United States', 'location.location.contains', 'Minnesota'],\n", + " ['Australia',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1900 Summer Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc3p9dt'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'Alvin and the Chipmunks: The Game'],\n", + " ['The Adventures of Rad Gravity', 'cvg.game_version.regions', 'Germany'],\n", + " ['Tijuana River', 'location.location.partially_containedby', 'California'],\n", + " ['1984 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United States of America'],\n", + " ['Tijuana River', 'common.topic.notable_types', 'River'],\n", + " ['France',\n", + " 'sports.sport_country.multi_event_tournaments_participated_in',\n", + " '2012 World Mountain Running Championships'],\n", + " ['United States Dollar',\n", + " 'finance.currency.countries_used',\n", + " 'American Samoa'],\n", + " ['United States Senator',\n", + " 'government.government_office_or_title.governmental_body_if_any',\n", + " 'United States Congress'],\n", + " ['United States of America',\n", + " 'base.litcentral.focal_location.priority_species',\n", + " 'm.063gzk6'],\n", + " ['United States of America',\n", + " 'location.location.primarily_containedby',\n", + " 'North America'],\n", + " ['2004 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['Saint John River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Germany'],\n", + " ['Pennsylvania', 'freebase.valuenotation.is_reviewed', 'Minimum wage'],\n", + " ['United States of America',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc3yvh2'],\n", + " ['Saint John River', 'geography.river.basin_countries', 'Canada'],\n", + " ['United States of America',\n", + " 'government.political_district.representatives',\n", + " 'm.03fx8bt'],\n", + " ['Sandia', 'base.militaryinfiction.location_in_fiction.contains', 'Sandia'],\n", + " ['m.063gzhq',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Endangered'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc534gx'],\n", + " ['m.04kc438',\n", + " 'military.casualties.type_of_casualties',\n", + " 'Military - Wounded'],\n", + " ['Tennessee', 'film.film_location.featured_in_films', 'Remote Area Medical'],\n", + " ['Marlene Bradford',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Abby Bradford'],\n", + " ['Guy', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['Canada', 'location.location.partially_contains', 'Similkameen River'],\n", + " ['Canada', 'location.location.partially_contains', 'Mount Saint Elias'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.12460zfqy'],\n", + " ['Nordstjernan', 'book.periodical.language', 'English Language'],\n", + " ['Cascade Range', 'location.location.partially_contained_by', 'm.0wg93tv'],\n", + " ['Team Garmin-Barracuda',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " [\"United States women's national rugby union team\",\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['France',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " 'The London 2012 Summer Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4rm_r'],\n", + " ['m.0b64vhy',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['U.S. federal district', 'freebase.type_hints.included_types', 'Location'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gender_balance_members_of_parliament',\n", + " 'g.1hhc37_k2'],\n", + " ['California', 'location.location.containedby', 'Contiguous United States'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_exported_to',\n", + " 'm.0493756'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Javier'],\n", + " ['Tennessee',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'South Dakota'],\n", + " ['Detroit River', 'location.location.partially_containedby', 'Canada'],\n", + " ['Mexico',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b658lk'],\n", + " ['1948 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'South Korea'],\n", + " ['United States of America',\n", + " 'government.governmental_jurisdiction.government_positions',\n", + " 'United States Senator'],\n", + " ['North America', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " 'United States national rugby union team'],\n", + " ['1936 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Canada'],\n", + " ['New York City', 'location.location.containedby', 'New York'],\n", + " ['m.0nfspz_',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'military.military_combatant.casualties',\n", + " 'm.043wpqn'],\n", + " ['m.0nfrxj5',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['m.0_fpyj5', 'measurement_unit.dated_integer.source', 'www.oica.net'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.096hjm7'],\n", + " ['United States of America',\n", + " 'location.location.time_zones',\n", + " 'Alaska Time Zone'],\n", + " ['m.063gzjl',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Endangered'],\n", + " ['Americas', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Tropical Storm Fay',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'Arkansas'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfrxxw'],\n", + " ['Beryl Andersen',\n", + " 'fictional_universe.fictional_character.gender',\n", + " 'Female'],\n", + " ['m.0gpry74',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['World Trade Organization',\n", + " 'organization.organization.founders',\n", + " 'United Kingdom'],\n", + " ['U.S. county',\n", + " 'base.aareas.schema.administrative_area_type.iso_country',\n", + " 'United States of America'],\n", + " ['m.052w0kt',\n", + " 'base.conservationaction.documented_priority_species.source',\n", + " 'Endangered and Threatened Wildlife and Plants; Review of Native Species That Are Candidates for Listing as Endangered or Threatened; Annual Notice of Findings on Resubmitted Petitions; Annual Description of Progress on Listing Actions'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.05nn1ry'],\n", + " ['m.0hnzf7y',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc438qq'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'National Observer'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nfsb7b'],\n", + " ['m.0wp701g',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Kettle River'],\n", + " ['Australia',\n", + " 'business.business_location.parent_company',\n", + " 'StayinFront, Inc.'],\n", + " [\"JoJo's Bizarre Adventure\",\n", + " 'fictional_universe.fictional_universe.locations',\n", + " 'Mexico'],\n", + " ['Montana',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Santa Cruz River', 'location.location.partially_containedby', 'Arizona'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfrxzw'],\n", + " ['Australia',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2008 Summer Olympics'],\n", + " ['m.06dn7g4',\n", + " 'base.athletics.athletics_championships_competition_athlete_relationship.medal',\n", + " 'Gold medal'],\n", + " ['Alaska', 'location.location.time_zones', 'Hawaii-Aleutian Time Zone'],\n", + " ['New Zealand',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2008 Summer Olympics'],\n", + " ['EverQuest II',\n", + " 'cvg.computer_videogame.publisher',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1q5k9wgqc'],\n", + " ['United States of America',\n", + " 'government.governmental_jurisdiction.government_positions',\n", + " 'President of the United States'],\n", + " ['2010 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'South Korea'],\n", + " ['Montana',\n", + " 'location.location.containedby',\n", + " 'United States, with Territories'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'Boulder Dash'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3dz46'],\n", + " ['m.0hnzf78',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['Alek Petrov',\n", + " 'fictional_universe.fictional_character.places_lived',\n", + " 'United States of America'],\n", + " ['Australia',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'World Trade Organization'],\n", + " ['m.0kmh3p1',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['Americas', 'location.location.contains', 'Navassa Island'],\n", + " ['Rocky Mountains', 'location.location.time_zones', 'Mountain Time Zone'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Wisconsin'],\n", + " ['m.0wg93y4',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['Frenchman River', 'location.location.containedby', 'North America'],\n", + " ['United States of America', 'cvg.cvg_publisher.games_published', 'The Pit'],\n", + " ['m.0nfsb72',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['South Korea',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'Earth'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'South Carolina'],\n", + " ['United States of America', 'location.country.languages_spoken', 'Lojban'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc489hf'],\n", + " ['m.0b64_23',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Silver medal'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_imported_from',\n", + " 'm.049377b'],\n", + " ['2010 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['Halls Stream', 'location.location.partially_containedby', 'Canada'],\n", + " ['Endangered',\n", + " 'base.conservationaction.priority_level.priorities_at_this_level',\n", + " 'm.063gzkz'],\n", + " ['United States of America',\n", + " 'base.militaryinfiction.location_in_fiction.contains',\n", + " 'Sandia'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b659gn'],\n", + " ['United States of America',\n", + " 'location.statistical_region.size_of_armed_forces',\n", + " 'g.1hhc3bjn0'],\n", + " ['Pacific Ocean', 'location.location.contains', 'Guam'],\n", + " ['United States of America',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc53825'],\n", + " ['Kentucky',\n", + " 'film.film_location.featured_in_films',\n", + " 'Shakespeare Behind Bars'],\n", + " ['Columbia River', 'common.topic.notable_types', 'River'],\n", + " ['Samoa Time Zone',\n", + " 'time.time_zone.locations_in_this_time_zone',\n", + " 'Palmyra Atoll'],\n", + " ['Contiguous United States', 'location.location.contains', 'Kentucky'],\n", + " ['Mother', 'common.topic.notable_types', 'Film character'],\n", + " ['United States of America',\n", + " 'fictional_universe.fictional_setting.fictional_characters_born_here',\n", + " 'T. Hawk'],\n", + " ['m.0nfs2__',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'm.0nfsrns'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Mount Alverstone'],\n", + " ['Cowlitz County', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['Arizona', 'common.topic.notable_types', 'US State'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Alabama'],\n", + " ['United States Dollar', 'finance.currency.countries_used', 'Guam'],\n", + " ['United States Army',\n", + " 'military.armed_force.military_combatant',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.0pl31jm'],\n", + " ['Saint John',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States, with Territories'],\n", + " ['Columbia River', 'location.location.partially_containedby', 'Canada'],\n", + " ['m.0493756',\n", + " 'location.imports_and_exports.currency',\n", + " 'United States Dollar'],\n", + " ['m.0nfs4nk',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['m.0nfsq07',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Tropical Storm Fay', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['United Kingdom',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2014 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Red River of the North'],\n", + " ['Guam', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_imported_from',\n", + " 'm.04fhph1'],\n", + " ['Natasha Andersen',\n", + " 'base.fictionaluniverse.deceased_fictional_character.place_of_burial',\n", + " 'United Kingdom'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs3dk'],\n", + " ['Canada',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1952 Summer Olympics'],\n", + " ['Vermont',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['La Opinión', 'common.topic.notable_types', 'Newspaper'],\n", + " ['United States of America',\n", + " 'government.governmental_jurisdiction.government_bodies',\n", + " 'United States Department of State'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs31z'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc42fsv'],\n", + " ['Albert Stroller', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['m.052w0jy',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc53ph3'],\n", + " ['Alek Petrov', 'common.topic.notable_types', 'Fictional Character'],\n", + " ['Thunderbird',\n", + " 'base.mystery.cryptid.area_of_occurrence',\n", + " 'United States of America'],\n", + " ['San Francisco', 'location.us_county.county_seat', 'San Francisco'],\n", + " ['Manhattan', 'location.us_county.county_seat', 'Manhattan'],\n", + " ['Federal government of the United States',\n", + " 'government.government.agency',\n", + " 'U.S. National Park Service'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Klaus'],\n", + " ['Buena Vista city',\n", + " 'location.us_county.hud_county_place',\n", + " 'Buena Vista city'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4118t'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'm.0nfsgbz'],\n", + " ['Illinois',\n", + " 'base.locations.states_and_provences.cities_within',\n", + " 'Belleville'],\n", + " ['m.0hnzfck',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['Hurricane Isabel',\n", + " 'meteorology.tropical_cyclone.affected_areas',\n", + " 'New York'],\n", + " ['France',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'World Trade Organization'],\n", + " ['Boulder Dash', 'cvg.game_version.developer', 'First Star Software'],\n", + " ['Arkansas',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['South Korea',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1948 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nfsbhv'],\n", + " ['m.0b64y35',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['X3: Terran Conflict',\n", + " 'base.wikipedia_infobox.video_game.platforms',\n", + " 'Microsoft Windows'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b64y5x'],\n", + " ['United Nations Command',\n", + " 'military.armed_force.military_combatant',\n", + " 'South Korea'],\n", + " ['Axiom Air Products',\n", + " 'organization.organization.locations',\n", + " 'United States of America'],\n", + " ['Serena Williams',\n", + " 'sports.tournament_event_competitor.events_competed_in',\n", + " 'm.0wjv1bf'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'MX vs. ATV Alive'],\n", + " ['T. Hawk',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'Mexico'],\n", + " ['United States of America',\n", + " 'location.statistical_region.labor_participation_rate',\n", + " 'g.1hhc42f3h'],\n", + " ['m.0wg942m',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.locations.countries.counties_within',\n", + " 'St. Clair County'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Delaware'],\n", + " ['United States of America',\n", + " 'base.athletics.athletics_country.championships_athletes_performances',\n", + " 'm.06dn7gb'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b64vhy'],\n", + " ['Canada', 'location.location.partially_contains', 'Frenchman River'],\n", + " ['United States of America',\n", + " 'organization.organization_member.member_of',\n", + " 'm.04dj56v'],\n", + " ['New York',\n", + " 'travel.travel_destination.tourist_attractions',\n", + " 'Niagara Falls'],\n", + " ['FOX Vakanties',\n", + " 'travel.tour_operator.travel_destinations',\n", + " 'United States of America'],\n", + " ['Ohio',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['Chamorro Time Zone',\n", + " 'time.time_zone.locations_in_this_time_zone',\n", + " 'United States of America'],\n", + " ['Mr Bradford',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['m.049y2k0',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['Boundary Butte', 'location.location.partially_contained_by', 'm.0wzh_k4'],\n", + " ['United Kingdom',\n", + " 'military.military_combatant.armed_forces',\n", + " 'United Nations Command'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'g.1hhc4_rzv'],\n", + " ['United States of America',\n", + " 'location.statistical_region.health_expenditure_as_percent_of_gdp',\n", + " 'g.1hhc4h4px'],\n", + " ['Wake Island', 'location.location.containedby', 'Pacific Ocean'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4y4c5'],\n", + " ['United States of America',\n", + " 'location.statistical_region.net_migration',\n", + " 'g.1q5jj_y4t'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wg93zc'],\n", + " ['Linnie Malcolm/Caroline Cresswell',\n", + " 'fictional_universe.fictional_character.place_of_birth',\n", + " 'United States of America'],\n", + " ['Rhode Island',\n", + " 'location.location.containedby',\n", + " 'Contiguous United States'],\n", + " ['Fortymile River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['Canada', 'location.location.partially_contains', 'Pembina River'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_imported_from',\n", + " 'm.049373t'],\n", + " ['1900 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0wp701g'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.049y2wb'],\n", + " ['m.063gzlb',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Endangered'],\n", + " ['Cuban Missile Crisis',\n", + " 'military.military_conflict.commanders',\n", + " 'm.0vmx6kz'],\n", + " ['Canada', 'location.location.partially_contains', 'Meduxnekeag River'],\n", + " ['United Kingdom',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1998 Winter Olympics'],\n", + " ['Saint Elias Mountains',\n", + " 'location.location.partially_containedby',\n", + " 'Alaska'],\n", + " ['Canada', 'location.location.time_zones', 'Pacific Time Zone'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc4p1wc'],\n", + " ['The Washington Post',\n", + " 'base.meedan.arabic_language_media_source.city',\n", + " 'Washington, D.C.'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'm.0nfs3bj'],\n", + " ['Contiguous United States', 'location.location.contains', 'Colorado'],\n", + " ['m.0wg93w8',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['New Mexico', 'freebase.valuenotation.is_reviewed', 'Minimum wage'],\n", + " ['m.049377t',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b65510'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Santa Cruz River'],\n", + " ['Japan',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Iowa'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Dora'],\n", + " ['France',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Le Moniteur Universel'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b659c6'],\n", + " ['m.0nfsrns',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'organization.organization_member.member_of',\n", + " 'm.04c_z8g'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1996 Summer Olympics'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wjpm7k'],\n", + " ['France', 'location.statistical_region.places_imported_from', 'm.049377t'],\n", + " ['United States Secretary of Housing and Urban Development',\n", + " 'government.government_office_or_title.jurisdiction',\n", + " 'United States of America'],\n", + " ['Aroostook River', 'geography.river.basin_countries', 'Canada'],\n", + " ['m.0563z51',\n", + " 'base.conservationaction.documented_priority_species.location',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'g.1hhc3zxqc'],\n", + " ['Serena Williams',\n", + " 'sports.tournament_event_competitor.country',\n", + " 'm.0112vj06'],\n", + " ['United States of America',\n", + " 'location.country.languages_spoken',\n", + " 'Saurashtra language'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4nftk'],\n", + " ['m.0b659jm',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Gold medal'],\n", + " ['m.0vmx69n',\n", + " 'military.military_command.military_conflict',\n", + " 'Cuban Missile Crisis'],\n", + " ['m.049377z',\n", + " 'location.imports_and_exports.exported_to',\n", + " 'United States of America'],\n", + " ['Michigan',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'cvg.computer_game_region.versions_released_in_this_region',\n", + " 'LEGO Pirates of the Caribbean: The Video Game'],\n", + " ['m.04kc46_', 'military.military_command.military_conflict', 'War of 1812'],\n", + " ['m.05kysvl',\n", + " 'organization.organization_membership.organization',\n", + " 'United Nations'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " \"United States women's national basketball team\"],\n", + " ['American Falls', 'location.location.partially_containedby', 'New York'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'g.1hhc488zp'],\n", + " ['Canada', 'location.location.partially_contains', 'Columbia River'],\n", + " ['Tijuana River',\n", + " 'geography.river.basin_countries',\n", + " 'United States of America'],\n", + " [\"United States women's national gymnastics team\",\n", + " 'sports.sports_team.gender',\n", + " 'Female'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Nevada'],\n", + " ['FOX Vakanties', 'travel.tour_operator.travel_destinations', 'New Zealand'],\n", + " ['Industrial Worker', 'book.periodical.language', 'English Language'],\n", + " ['United States of America',\n", + " 'base.mystery.cryptid_area_of_occurrence.cryptid_s_found_here',\n", + " 'Chupacabra'],\n", + " ['U.S. independent city',\n", + " 'base.aareas.schema.administrative_area_type.subdivides_type',\n", + " 'U.S. state'],\n", + " ['New Zealand',\n", + " 'organization.organization_founder.organizations_founded',\n", + " 'World Trade Organization'],\n", + " ['Star Wars: Republic Commando',\n", + " 'cvg.computer_videogame.publisher',\n", + " 'United States of America'],\n", + " ['m.0b64y5n',\n", + " 'sports.competitor_competition_relationship.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Florida'],\n", + " ['California',\n", + " 'film.film_location.featured_in_films',\n", + " 'Have Tig at Your Party'],\n", + " ['United States of America',\n", + " 'location.statistical_region.net_migration',\n", + " 'g.1s0636sg5'],\n", + " ['Taiwan', 'location.country.form_of_government', 'Constitutional republic'],\n", + " ['m.0nfry06',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Minnesota',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['m.06dn7lv',\n", + " 'base.athletics.athletics_championships_competition_athlete_relationship.championships',\n", + " '2007 World Championships in Athletics'],\n", + " ['m.0nfshq4',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Military expenditure as percentage of GDP, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Kansas',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b7bysn'],\n", + " ['m.04fl3np',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc4fk_j'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b653p5'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf3s'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gross_savings_as_percent_of_gdp',\n", + " 'g.1hhc3b9p_'],\n", + " ['2014 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Germany'],\n", + " ['United States of America',\n", + " 'base.litcentral.focal_location.priority_species',\n", + " 'm.063gzl2'],\n", + " ['2010 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Mexico'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3vpv6'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'North Dakota'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Iowa'],\n", + " ['United States of America', 'symbols.flag_referent.flag', 'm.07sgsm_'],\n", + " ['Guam', 'location.country.official_language', 'English Language'],\n", + " ['Washington, D.C.',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Chronicle of Higher Education'],\n", + " ['Maine', 'common.topic.notable_types', 'US State'],\n", + " ['Michigan', 'location.location.partially_contains', 'St. Marys River'],\n", + " ['m.063gzll',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Endangered'],\n", + " ['Crimson Viper', 'common.topic.notable_types', 'Film character'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Massachusetts'],\n", + " ['United Kingdom',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1992 Winter Olympics'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Irene–Olivia'],\n", + " ['United States of America',\n", + " 'organization.organization_member.member_of',\n", + " 'm.04ddsbr'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wg93v9'],\n", + " ['Texas',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Vermont'],\n", + " ['Saint Francis River', 'geography.river.basin_countries', 'Canada'],\n", + " ['m.0nfspz7',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc42fhx'],\n", + " ['Samoa Time Zone',\n", + " 'time.time_zone.locations_in_this_time_zone',\n", + " 'United States of America'],\n", + " ['Wyoming',\n", + " 'location.administrative_division.country',\n", + " 'United States of America'],\n", + " ['Priority 3',\n", + " 'base.conservationaction.priority_level.priorities_at_this_level',\n", + " 'm.052w0l1'],\n", + " ['Contiguous United States', 'location.location.contains', 'Utah'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.developer',\n", + " \"Traveller's Tales\"],\n", + " [\"United States men's national inline hockey team\",\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['Canada', 'location.location.partially_contains', 'Kootenay River'],\n", + " ['m.0hnzf36',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['United States Minor Outlying Islands',\n", + " 'base.aareas.schema.administrative_area.administrative_children',\n", + " 'Palmyra Atoll'],\n", + " ['m.0hnzf57',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['United States of America',\n", + " 'location.statistical_region.trade_balance_as_percent_of_gdp',\n", + " 'g.1hhc3h6v4'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3kwt0'],\n", + " ['m.0dkbjtj',\n", + " 'government.government_position_held.appointed_by',\n", + " 'Harry S. Truman'],\n", + " ['m.0hnzfdr',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['m.0wg940v',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'government.political_district.representatives',\n", + " 'm.03fx8c3'],\n", + " ['United States of America',\n", + " 'location.statistical_region.net_migration',\n", + " 'g.1q5jxqz89'],\n", + " ['United States, with Territories',\n", + " 'base.aareas.schema.earth.sovereign_domain.sovereign_state',\n", + " 'United States of America'],\n", + " ['1952 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'France'],\n", + " ['m.0dm4wc4',\n", + " 'location.imports_and_exports.exported_to',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wkfv0n'],\n", + " ['Canada', 'location.location.partially_contains', 'Mount Fairweather'],\n", + " ['Texas',\n", + " 'base.aareas.schema.administrative_area.administrative_area_type',\n", + " 'U.S. state'],\n", + " ['United States of America',\n", + " 'fictional_universe.fictional_setting.fictional_characters_born_here',\n", + " 'Bryan Fury'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc4dq9j'],\n", + " ['m.0wg93_k',\n", + " 'location.partial_containment_relationship.partially_contained_by',\n", + " 'United States of America'],\n", + " ['Canada', 'location.location.partially_contains', 'Red River of the North'],\n", + " ['Washington, D.C.',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Washington Times'],\n", + " ['Georgia', 'location.location.time_zones', 'Eastern Time Zone'],\n", + " ['Zaman', 'book.newspaper.circulation_areas', 'New York'],\n", + " ['United States Minor Outlying Islands',\n", + " 'location.location.contains',\n", + " 'Palmyra Atoll'],\n", + " ['United States of America',\n", + " 'base.database.database_topic.database_s_for_this_topic',\n", + " 'data.gov'],\n", + " ['Battle of New Orleans', 'time.event.included_in_event', 'War of 1812'],\n", + " ['Meduxnekeag River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf9q'],\n", + " ['United States of America',\n", + " 'film.film_location.featured_in_films',\n", + " 'Remote Area Medical'],\n", + " ['United States of America',\n", + " 'location.statistical_region.lending_interest_rate',\n", + " 'g.1hhc4k9ys'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b659jm'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2004 Summer Olympics'],\n", + " ['m.0g9jx_k',\n", + " 'base.firsts.first_achievement.category',\n", + " 'United States of America'],\n", + " ['Johnston Atoll', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['m.052w0l1',\n", + " 'base.conservationaction.documented_priority_species.priority_level',\n", + " 'Priority 3'],\n", + " ['United States of America',\n", + " 'base.morelaw.plaintiff.lawsuits',\n", + " 'United States v. Libby'],\n", + " ['United States Marine Corps Forces, Europe',\n", + " 'military.military_unit.place_of_origin',\n", + " 'United States of America'],\n", + " ['Arizona',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['1936 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'Taiwan'],\n", + " ['Hurricane Paul', 'meteorology.tropical_cyclone.affected_areas', 'Mexico'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'North Star'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.04h_h48'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc3c16t'],\n", + " ['Mount Fairweather', 'location.location.partially_containedby', 'Alaska'],\n", + " ['United States of America',\n", + " 'meteorology.cyclone_affected_area.cyclones',\n", + " 'Hurricane Newton'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'Maine'],\n", + " ['Connecticut', 'location.location.containedby', 'United States of America'],\n", + " ['The Washington Times', 'book.periodical.language', 'English Language'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc45tlm'],\n", + " ['United States of America',\n", + " 'sports.sports_team_location.teams',\n", + " \"United States men's national inline hockey team\"],\n", + " ['Australia',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Draugas'],\n", + " ['Milk River', 'location.location.partially_contained_by', 'm.0wg9400'],\n", + " ['Battle of Iwo Jima', 'time.event.included_in_event', 'World War II'],\n", + " ['Mount Fairweather',\n", + " 'location.location.containedby',\n", + " 'Pacific Coast Ranges'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Texas'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc3mbvy'],\n", + " ['United States of America',\n", + " 'base.firsts.first_achievement_category.firsts',\n", + " 'm.04x258g'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc3bf6c'],\n", + " ['Mexico',\n", + " 'fictional_universe.fictional_setting.fictional_characters_born_here',\n", + " 'T. Hawk'],\n", + " ['1984 Summer Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'United Kingdom'],\n", + " ['Mississippi', 'freebase.valuenotation.has_no_value', 'Minimum wage'],\n", + " ['Washington, D.C.',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Missouri', 'common.topic.notable_types', 'US State'],\n", + " ['m.0nfsk7w',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Taiwan', 'location.statistical_region.places_exported_to', 'm.049373t'],\n", + " ['United States of America',\n", + " 'location.country.first_level_divisions',\n", + " 'West Virginia'],\n", + " ['Gazette van Detroit', 'common.topic.notable_types', 'Newspaper'],\n", + " ['m.0b6599q',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2009 World Championships in Athletics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.co2_emissions_per_capita',\n", + " 'm.0nfrzy8'],\n", + " ['m.012cdjgk',\n", + " 'religion.religious_organization_leadership.jurisdiction',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'The Village Voice'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.game',\n", + " 'Lego Pirates of the Caribbean: The Video Game'],\n", + " ['United States of America',\n", + " 'sports.sport_country.athletic_performances',\n", + " 'm.0b64zy1'],\n", + " ['United States of America',\n", + " 'location.statistical_region.market_cap_of_listed_companies_as_percent_of_gdp',\n", + " 'g.1hhc3sz9r'],\n", + " ['United States of America',\n", + " 'base.sharing.sharing_location.shared_here',\n", + " 'm.0gw7mgc'],\n", + " ['m.049377n',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.places_imported_from',\n", + " 'm.049378y'],\n", + " ['Navassa Island',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States, with Territories'],\n", + " ['Ruth Osborne',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Celia Osborne'],\n", + " ['Rufus', 'fictional_universe.fictional_character.gender', 'Male'],\n", + " ['m.04xd0mv',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " ['English Language', 'language.human_language.main_country', 'Australia'],\n", + " ['South Korea',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2010 Winter Olympics'],\n", + " ['United States of America',\n", + " 'fictional_universe.fictional_setting.characters_that_have_lived_here',\n", + " 'Claire Underwood'],\n", + " ['2014 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'South Korea'],\n", + " ['Hunchak', 'common.topic.notable_types', 'Newspaper'],\n", + " ['United States of America',\n", + " 'location.statistical_region.religions',\n", + " 'm.03q1lvh'],\n", + " ['m.0nfspzh',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['m.07sgsm_', 'symbols.flag_use.flag_user', 'United States of America'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Stars and Stripes'],\n", + " ['m.0nfshr0',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Military expenditure as percentage of GDP, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['San Francisco', 'base.biblioness.bibs_topic.subsumes', 'San Francisco'],\n", + " ['m.0bp733g',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['Gazette of the United States', 'common.topic.notable_types', 'Newspaper'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3k0pt'],\n", + " ['m.03fx87q',\n", + " 'government.government_position_held.office_position_or_title',\n", + " 'President of the United States'],\n", + " ['South Dakota', 'common.topic.notable_types', 'US State'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc3b2s9'],\n", + " ['Colorado', 'location.location.time_zones', 'Mountain Time Zone'],\n", + " ['m.0nfs32d',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'military.military_combatant.military_commanders',\n", + " 'm.0b68f31'],\n", + " ['Germany',\n", + " 'sports.sport_country.multi_event_tournaments_participated_in',\n", + " '2012 World Mountain Running Championships'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nfs4qb'],\n", + " ['Colorado', 'location.location.partially_contains', 'Rio Grande'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc3_483'],\n", + " ['m.0gps9gs',\n", + " 'military.casualties.military_conflict',\n", + " 'Attack on Pearl Harbor'],\n", + " ['United States of America',\n", + " 'location.statistical_region.military_expenditure_percent_gdp',\n", + " 'm.0nfshnd'],\n", + " ['Germany',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1900 Summer Olympics'],\n", + " ['StayinFront, Inc.',\n", + " 'organization.organization.locations',\n", + " 'United Kingdom'],\n", + " ['Minnesota', 'location.location.containedby', 'Contiguous United States'],\n", + " ['m.04fl3qp',\n", + " 'location.imports_and_exports.exported_to',\n", + " 'United States of America'],\n", + " ['m.04dj56v',\n", + " 'organization.organization_membership.member',\n", + " 'United States of America'],\n", + " ['New River', 'geography.river.basin_countries', 'Mexico'],\n", + " ['United States of America',\n", + " 'location.statistical_region.cpi_inflation_rate',\n", + " 'g.1hhc37csd'],\n", + " ['m.011v4f3m',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Silver medal'],\n", + " ['Alabama',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['United States Davis Cup team',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1984 Summer Olympics'],\n", + " ['m.0hnzf6q',\n", + " 'measurement_unit.adjusted_money_value.adjustment_currency',\n", + " 'United States Dollar'],\n", + " ['m.0nfsk6w',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.statistical_region.part_time_employment_percent',\n", + " 'g.1hhc4t_km'],\n", + " ['United States of America',\n", + " 'base.locations.countries.states_provinces_within',\n", + " 'Virginia'],\n", + " ['Dogs', 'film.film.language', 'English Language'],\n", + " ['m.0kmhk5s',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Silver medal'],\n", + " ['River', 'freebase.type_hints.included_types', 'Location'],\n", + " ['Counter-Strike: Global Offensive',\n", + " 'common.topic.notable_types',\n", + " 'Video Game Version'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_per_capita_in_ppp_dollars',\n", + " 'm.0nfsbjb'],\n", + " ['Patrica Spade', 'common.topic.notable_types', 'Fictional Character'],\n", + " ['United States of America',\n", + " 'organization.organization.parent',\n", + " 'm.09qmnrh'],\n", + " ['Jane Andersen',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Sarah Andersen'],\n", + " ['m.0nfsk93',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['Detroit River',\n", + " 'location.location.partially_containedby',\n", + " 'United States of America'],\n", + " ['Marshall Islands', 'common.topic.notable_types', 'Country'],\n", + " ['The Chronicle of Higher Education',\n", + " 'book.newspaper.circulation_areas',\n", + " 'Washington, D.C.'],\n", + " ['United States, with Territories', 'common.topic.notable_types', 'Country'],\n", + " ['Texas', 'location.location.partially_contains', 'Rio Grande'],\n", + " ['Vietnam War', 'military.military_conflict.casualties', 'm.043wptp'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.pertinent_type',\n", + " 'U.S. county'],\n", + " ['American Samoa',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States, with Territories'],\n", + " ['m.02hvy7h',\n", + " 'aviation.aircraft_ownership_count.aircraft_owner',\n", + " 'United States of America'],\n", + " ['Nevada',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States of America'],\n", + " ['m.0pl31jm',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['Americas', 'location.location.time_zones', 'Central Time Zone'],\n", + " ['Canada',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '2002 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gdp_real',\n", + " 'm.0hnzf20'],\n", + " ['Canada',\n", + " 'olympics.olympic_participating_country.olympics_participated_in',\n", + " '1948 Winter Olympics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'g.1245_d8zp'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Mississippi'],\n", + " ['m.0wg940n',\n", + " 'location.partial_containment_relationship.partially_contains',\n", + " 'Saint Lawrence River'],\n", + " ['Saint Lawrence River', 'geography.river.basin_countries', 'Ohio'],\n", + " ['United States of America',\n", + " 'location.statistical_region.merchandise_trade_percent_of_gdp',\n", + " 'g.1hhc4s8dy'],\n", + " ['Viceroy Miami', 'location.location.containedby', 'Florida'],\n", + " ['Washington', 'location.location.containedby', 'United States of America'],\n", + " ['LEGO Pirates of the Caribbean: The Video Game',\n", + " 'cvg.game_version.developer',\n", + " \"Traveller's Tales\"],\n", + " ['m.0nfsgbr',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Military expenditure as percentage of GDP, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Iowa', 'common.topic.notable_types', 'US State'],\n", + " ['United Nations', 'organization.organization.founders', 'Canada'],\n", + " ['Hurricane Javier', 'common.topic.notable_types', 'Tropical Cyclone'],\n", + " ['David A. Bray',\n", + " 'people.person.place_of_birth',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.consumer_price_index',\n", + " 'g.1hhc4bpj_'],\n", + " [\"Starlight Children's Foundation\",\n", + " 'organization.organization.geographic_scope',\n", + " 'United Kingdom'],\n", + " ['m.0hnzf6q',\n", + " 'measurement_unit.adjusted_money_value.source',\n", + " 'World Bank, World Development Indicators'],\n", + " ['United States of America',\n", + " 'location.location.partiallycontains',\n", + " 'm.0wzh_k4'],\n", + " ['TV Character', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['Washington, D.C.',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Zaman'],\n", + " ['Le Moniteur Universel',\n", + " 'book.newspaper.circulation_areas',\n", + " 'United States of America'],\n", + " ['Sam Seaborn', 'common.topic.notable_types', 'TV Character'],\n", + " ['Saint Lawrence River', 'common.topic.notable_types', 'River'],\n", + " ['Nebraska',\n", + " 'base.biblioness.bibs_location.country',\n", + " 'United States of America'],\n", + " ['Counter-Strike: Global Offensive',\n", + " 'common.topic.notable_types',\n", + " 'Video Game Version'],\n", + " ['m.049375j',\n", + " 'location.imports_and_exports.imported_from',\n", + " 'United States of America'],\n", + " [\"Tom Clancy's Ghost Recon: Shadow Wars\",\n", + " 'cvg.game_version.regions',\n", + " 'United States of America'],\n", + " ['North Atlantic Treaty Organization (NATO)',\n", + " 'organization.organization.founders',\n", + " 'France'],\n", + " ['Earth', 'base.locations.planets.countries_within', 'France'],\n", + " ['Pacific Coast Ranges', 'location.location.contains', 'Mount Hubbard'],\n", + " ['m.0nfsp_r',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'Gross National Income in PPP dollars, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America', 'common.topic.image', 'united-states.jpg'],\n", + " ['Betty Lou Who',\n", + " 'fictional_universe.fictional_character.children',\n", + " 'Drew Lou Who'],\n", + " ['North Dakota',\n", + " 'location.location.partially_contains',\n", + " 'Red River of the North'],\n", + " ['m.048z_91',\n", + " 'military.military_command.military_combatant',\n", + " 'United States of America'],\n", + " ['m.0nfry2f',\n", + " 'measurement_unit.dated_metric_ton.source',\n", + " 'CO2 emissions per capita, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['Washington',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.124619h4w'],\n", + " ['Johnston Atoll',\n", + " 'location.location.time_zones',\n", + " 'Hawaii-Aleutian Time Zone'],\n", + " ['m.04x258g',\n", + " 'base.firsts.first_achievement.category',\n", + " 'United States of America'],\n", + " ['New Mexico', 'location.location.partially_contains', 'Rio Grande'],\n", + " ['Illinois',\n", + " 'location.administrative_division.first_level_division_of',\n", + " 'United States of America'],\n", + " ['Washington, D.C.',\n", + " 'location.location.containedby',\n", + " 'United States, with Territories'],\n", + " ['m.0nfs3dk',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Internet users as percentage of population, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Alaska'],\n", + " ['Japan', 'location.statistical_region.places_imported_from', 'm.0493792'],\n", + " ['Executive Office of the President of the United States',\n", + " 'government.government_agency.jurisdiction',\n", + " 'United States of America'],\n", + " ['m.0nfshsk',\n", + " 'measurement_unit.dated_percentage.source',\n", + " 'Military expenditure as percentage of GDP, World Development Indicators and Global Development Finance, World Bank'],\n", + " ['United States national cricket team',\n", + " 'sports.sports_team.location',\n", + " 'United States of America'],\n", + " ['United States, with Territories',\n", + " 'base.aareas.schema.administrative_area.subdividing_place',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.statistical_region.long_term_unemployment_rate',\n", + " 'g.1hhc4w10j'],\n", + " ['Taiwan', 'location.statistical_region.places_exported_to', 'm.049376_'],\n", + " ['United States of America',\n", + " 'base.aareas.schema.administrative_area.pertinent_type',\n", + " 'U.S. state'],\n", + " ['Barnstable County',\n", + " 'location.administrative_division.second_level_division_of',\n", + " 'United States of America'],\n", + " ['United States of America',\n", + " 'location.location.partially_contains',\n", + " 'Pend Oreille River'],\n", + " ['United States of America', 'location.location.adjoin_s', 'm.02wtp8v'],\n", + " ['United States of America',\n", + " 'location.statistical_region.gni_in_ppp_dollars',\n", + " 'm.0nfspz_'],\n", + " ['The Adventures of Rad Gravity',\n", + " 'cvg.game_version.regions',\n", + " 'United States of America'],\n", + " ['m.0nfsk73',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States of America',\n", + " 'location.country.administrative_divisions',\n", + " 'Marshall Islands'],\n", + " ['m.0b64vqj',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Bronze medal'],\n", + " ['2010 Winter Olympics',\n", + " 'olympics.olympic_games.participating_countries',\n", + " 'New Zealand'],\n", + " ['m.0nfsb54',\n", + " 'measurement_unit.dated_money_value.currency',\n", + " 'United States Dollar'],\n", + " ['United States national speedway team',\n", + " 'common.topic.notable_types',\n", + " 'Sports Team'],\n", + " ['United States of America',\n", + " 'periodicals.newspaper_circulation_area.newspapers',\n", + " 'Weekly World News'],\n", + " ['Natasha Andersen',\n", + " 'fictional_universe.fictional_character.parents',\n", + " 'Jane Andersen'],\n", + " ['Northern Mariana Islands',\n", + " 'base.aareas.schema.administrative_area.administrative_parent',\n", + " 'United States, with Territories'],\n", + " ['36th Flying Training Wing',\n", + " 'military.military_unit.place_of_origin',\n", + " 'United States of America'],\n", + " ['Nevada',\n", + " 'base.locations.states_and_provences.country',\n", + " 'United States of America'],\n", + " ['France', 'base.locations.countries.planet', 'Earth'],\n", + " ['Australia', 'freebase.valuenotation.is_reviewed', 'Time zone(s)'],\n", + " ['Northern Mariana Islands',\n", + " 'location.country.form_of_government',\n", + " 'Presidential system'],\n", + " ['United States of America',\n", + " 'location.statistical_region.internet_users_percent_population',\n", + " 'g.1245ywrlb'],\n", + " ['m.049377z',\n", + " 'location.imports_and_exports.currency',\n", + " 'United States Dollar'],\n", + " ['2012 World Mountain Running Championships',\n", + " 'sports.multi_event_tournament.participating_countries',\n", + " 'Australia'],\n", + " ['m.0b6598c',\n", + " 'sports.competitor_competition_relationship.tournament',\n", + " '2009 World Championships in Athletics'],\n", + " ['United States of America',\n", + " 'location.statistical_region.high_tech_as_percent_of_manufactured_exports',\n", + " 'g.1hhc4krtn'],\n", + " ['m.0b64zy1',\n", + " 'sports.competitor_competition_relationship.medal',\n", + " 'Gold medal'],\n", + " ...]]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.raw_dataset[:10]['graph']" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With these questions and triplets, we want to:\n", + "1. Consolidate all the relations in these triplets into a Knowledge Graph\n", + "2. Create a FeatureStore that encodes all the nodes and edges in the knowledge graph\n", + "3. Create a GraphStore that encodes all the edge indices in the knowledge graph" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch_geometric.nn.nlp import SentenceTransformer\n", + "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append('..')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to create a remote backend, we need to define a FeatureStore and GraphStore locally, as well as a method for initializing its state from triplets:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader\n", + "\n", + "# We define this GraphStore to sample the neighbors of a node locally.\n", + "# Ideally for a real remote backend, this interface would be replaced with an API to a Graph DB, such as Neo4j.\n", + "from rag_graph_store import NeighborSamplingRAGGraphStore\n", + "\n", + "# We define this FeatureStore to encode the nodes and edges locally, and perform appoximate KNN when indexing.\n", + "# Ideally for a real remote backend, this interface would be replaced with an API to a vector DB, such as Pinecone.\n", + "from rag_feature_store import SentenceTransformerFeatureStore" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(model_name=\"sentence-transformers/all-roberta-large-v1\").to(device)\n", + "\n", + "backend_loader: RemoteGraphBackendLoader = create_remote_backend_from_triplets(\n", + " triplets=all_triplets, # All the triplets to insert into the backend\n", + " node_embedding_model=model, # Embedding model to process triplets with\n", + " node_method_to_call=\"encode\", # This method will encode the nodes/edges with 'model.encode' in this case.\n", + " path=\"backend\", # Save path\n", + " pre_transform=preprocess_triplet, # Preprocessing function to apply to triplets before invoking embedding model.\n", + " node_method_kwargs={\"batch_size\": 256}, # Keyword arguments to pass to the node_method_to_call.\n", + " graph_db=NeighborSamplingRAGGraphStore, # Graph Store to use\n", + " feature_db=SentenceTransformerFeatureStore # Feature Store to use\n", + " ) \n", + "# This loader saves a copy of the processed data locally to be transformed into a graphstore and featurestore when load() is called.\n", + "feature_store, graph_store = backend_loader.load()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have initialized our remote backends, we can now retrieve from them using a Loader to query the backends, as shown in this diagram:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAAGRCAYAAADmVIAxAAAAAXNSR0IArs4c6QAAAFBlWElmTU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAACDaADAAQAAAABAAABkQAAAABhaUSHAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgoZXuEHAABAAElEQVR4AeydBXzV1f+HtwGiqCh2oAwURSUUEwNmtwIqtmJ3d2P/zL/dAXaD3Yrd3ckEW0FURCW2//PcfQ9+vd7FhbsxtvN+vZ578nvic/Le3W1FRVHRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtECTdkCxU25c7Fv0QLRAk3XAm3atPmZ3s3ddHsYexYt0GAWGD1+/Ph5Gqy2WFG0QLRAtEBDW4BLQ2VD1xnrixZoihbIZy2VNEUDxD5FC0QLRAtEC0QLRAsU3gItayoyfvxXk3ViWrRAXhaIH//lZa6YOVogWqAxWqDGSwMNnpufc8TvPTTGkYttmqEskM/HfzNUx2JjowWiBZqVBeKPJ5rVcMfORgtEC0QLRAtEC0y9BeKlYeptF5+MFogWiBaIFogWaFYWiJeGZjXcsbPRAtEC0QLRAtECU2+BeGmYetvFJ6MFogWiBaIFogWalQXipaFZDXfsbLRAtEC0QLRAtMDUWyBeGqbedvHJaIFogWiBaIFogWZlgXhpaFbDHTsbLRAtEC0QLRAtMPUWiJeGqbddfDJaIFpgxrNAN5rcD9aDdvXU/DkotwwWqmP5rci3MmyTuC3q+FxDZOtBJVvASlDov9mzBmXOBNNbq9OA1jAnrDAdGrNaUn99VN2ZQhetj4Jzlhn/IE1Os8TIaIG8LRDXUt4mq/WBPG26CAW+Av6/isCf+PeEQstDyDr2qkPBtuvTJH9o10eEF06eXRB3CCydhBvKKaWiV8G23ASfwOswPxRK4ygo9LNQZU5NOb/xkONQBm/B1Go/Htx4Kh4ewzPWXx+6mEKPqq3gfNZS/KThH2t60yyFWf6JajS+9rRk5kbTmtiQaIEZywK+Q74PesKR0AX6w09wWRLGmS46gloXB9/Nd4TjwPYdD2o32AnsQ0PJc2EoPA5LwQ5gm76Ey6GpajgdW24aOrcpz3qONF/lcfvwI51wS067TryplR/ReVPvNLUF1PE5Lwk3w99g2ycm4Ta4ygXkuxEXdkPKS8L58AuMgN/hCjC+UBpLQR0KVdg0lDOaZx1n36G9Nw3llPHsalPx/I8844Zdb8pjLdVbG5pawXnYdC367to+L8sGqxDeGuZL4j/GvRKcg77jdA9yDTo/foWXwY/p1f/gDfBd3HfgO3EPDeU8tj4PWJ9xDV8Huf5s/9XEm/cg8ONx69wMusOKMAZM/xrWBC8Ph8FH4Pp9GDzY1dowAmybbT4ZZoPrwXI+g0OgNmmvH8Afm6Tlj1tMUwvA3fAFWNet0AbUO3Ao2OaRUN2nOX7ScBGUw/sQ7Ic3My7GjQLrmRey1ZuIG+FF+AC03QAIz92DP4ztqfjPhTdAG50H5le/ge/0nQ8PgpoZQtucFwcaiWzHbRD6fRf+tnAAjAftZhu0nfWVg5etQeDYqZ7wEnwDV4DjmOuTBufRC6AdbfMxoLzY2IYbwGft08qgZgXH27J99hE4CmpUHmupxnKK8ihoBUqqhOfgpBRb459aHc6Dltl9aguo43PHJ/Wcj7szOBjWewaoQ8HwsgYaUHdQlxPYSap0X4GLDRRITrgOBSprWooZzcOdYA5YcRoKeoBnXbD5yk2vY74P5ZM/j7WUT7HNOm8eNj0CQ7mGN00M5iZflqIUv/oTzOdG7jMbgQfBCbAteHFwE1Ye9hXwJOwG/ojB5xeFcGnwwHQPvAXS9ROcom74XIemTwI3+r3BA82yngLTLoXOcGASvhv3YPgevoM5YRMwr/P5bFgbroe/wDKNM70/1CQvMI/WlIG0W+GsJM/suF60tJH6Dc4D+6At/oZZIFvjiLgEPEyXh9HgxcQ9wEPPPUFp/wczvn+/rEdQm9nvZWAF+BYWA3U8PJTxVX2i9An++aANPAP7gLK9Htpl8Bao0+BemBnmhXLoBdfCRWCbLec12A2UcyPY9hT894P9ng2Gw57QEr6CsE9pa8fE+tMyn2O7ThK5FK59nQdsh3NvU1Anw5MZX1HRObh3gc8vCT/BUVCj8lhLNZYzNZeGM6spcW7iPZSfhv+D+SFI4w0D08yzICwOb4DGNG1Z2AmuBwdRbQ+GHbjuiX9L3Idhd1AOkgN5B/SGXLqOSOtxIJTlnwQ7gpPvdTDdCdQT1Gpg3Q7UIGgLqhsYbzucrNav9gDbcSf0gdq0DBnGw1xZGXsQPiKJc1IcBvbXsg+FElCXgv0dmuDCzaWxRO4KLsgboAsEOUmHwBNwNFhftsxzMDhuN8NMYBnp51oRVo6XY6197oNtIMjNohN0htNCJK55HP/bIdjNjehAsM2PggvCuI1hFAyHMlD9wYVrfaYHzYXnXHgc7P+P0BHqTQVblPXWwhmv4Dxsegy9q4QwB1ZNwsaJm676E97K+P55aY3X9X4k/AzuS+pq8NlSUM5tw65115v+Y0GtAYb3NZBD8xNn+a61v8C8F4M6Hgy7J6gP4CtwzquBYLruJok/1FtMeBy8CmUJXnxug5p0KIkP1ZQhSZsDd3lwbb8He4H6DRbL+Kpe/sBZIBUOXtuWjn+A8EA4G9xvyxIct0nQBtJaj4D2CDoLj2u9LGEj3MkwK1wGh0PQtnjcO5XtXQTKIIy//UmfGfMS1p7K/b4nWMabcAioR6B/xldU9DGuY1qWcAau6SvCCAgqwfMLWH+23DvnAufT/jAeOkEv+AKCnJ/vJIFPcA0HXY7nqBCozs1jLRXZ4ELKzp2UQuO2gMdhJ9DA/eBp0CBrghPYSW+ahrkEXKhzgvLm6WRZBQaCz6mVYSCYt33ivx7XOi3PhXMl/ACW9SRYRrasvwJehI/gTLgfbgTLbgcqtGN9/M+AA+MmYj32ZyZYGAbCYHBBWe7RcBX8CLbtCVgVapLp2mNMViYnxjlJnIer9jsMnJAe3puCcjK7QP4Hz4KL0fblkuNyDLwET4Ft1J72ybYeBD3AccnWAkRY90QwrwtreOI/ELc7hOccr0vhTrDtp0BfSGt+ApsnEbvgHg9ng+PoxaELGGc/XZCWsQcMABf51zAcPoHtwXocT212PriJqHuhJRwBjuM8ENV0LfBB0rUVE/d9XNeOcztbI1MRzlnDQ6AU3LTdW9IakwRc32r2KifzGtJcH6q4yvnX64GENgPn+TqwMJTDjpBLHiLfweQkcVTizp24OqEPM+P3wOwAJyW8ievzNeljEpfKkWFe4o4D9+C9wP3yROgMv0K6f9oqyLam00K8rs8F6dd+7WAO6JOwAu6p0AKy9UsqwufaQnjO8XaPKAHlfh30G570WIX44HpmjA0B3J+gEnYG7TMIlgbz5OqbbekEoS1/478bbJ/9DPKMSPchxFum58YbsB/Yh0kQ6voDf5BlhHjtli4/3eeQv/7cPG4fDqoGzaY9cWsm8Qfgqk3BfOuDk3o50JBrwFegkZQHmfm6G0CXgeHZDaCLwLCD42Gg/3wI+h7P80nAST4GrkvC2U4ZEXeCxrYcByG096gkbllc9Sw4yE4qdQj4zNawQeK/ADfoWzwvJgHbMRoGJ+HqHMt8pLrEJH4Z3LBRLI7/aXByKfsa2mt4HLgZZWssEU78oCfwbAfHwr1QmtAT10nveKXl2H6dinDM7oPShOVww3OO1xkQtAueYUlAm7jAVgcPf+XYbZ7xVb2shrMguJnNVxWVeUY7HZGEH8AdkPhfwXUMSxMcR9vmBmd9YQNqjf936Aj1pjzWUr21oakVnIdNnbcj4Fdwr1ALQNhTTs7EVH3S4D4QNBiPa7stOF9GwuugrgbTXC/qRDBs+c5j/R6sahUwHNancUHOc9dI7yTCdfAjlCdhD2mf7ZGEne/jYbEkfCmu6e6fmyR+96Kgcjzvg+23H6fAqqB6QXqfyETyor2+gW1DROK6fl9N/K6hPolfx/W2TxL+Dde1GpQdDvHuSx7sqgQ+hDXB9fwgBC2Ex7rNk9Z6BLRf0OF4HgoBXNsQnnOsz0ulnYD/yiRs+xaBMngL1NOwfcZX9XIxzsYwCjasisq8mu+wJPww7haJ33btmfh1fHYALAzuN7ODmhP+ButPa3kC7s+zJJFz406CxaEXvAtB6fBTRO4QEnCHgntfjcpjLeX8yLnGwmtJ/B/px2TlWSsJX4QrQW7STubTYW1wwrSB7IlBVE7lyvdektODYP4EF1RQp+BJuavhnwBbgWU6aW8FJ9XFkC0X62fggKqwiVh2mHChHa2IWzChtnaQbYpG4lt0Sugfj4vZ9g4HN4ArwHq143xQDEEu6iD7Z99yybqCXBBupk7g7nA9BL2Ixzr/ChGJ+20q7HPdINdzZvvSl0Rf47oZVCcXl+0JeiHxGO8GsAR8BPNCut8EM2rP607QvyqYeXXcfP47mJyJqVqw6T4k0dFpQhZwzrq+vQinDyO76Jy4X08OvU3cznAfzApho8c7RefjOxaWBg/Ox2AVqKuOJuOj8AyMB/dA5+ZhoFwnynL3But6Et6BMeCauw6eAy8NKr3XnEB4CLgnuR+5LqxP3Qmuyd4GUtJerp27YH34AtwP+sA6oNyvdwX3HQ9R9zkPwHx1AQ+4X1iP63A4vAv7wOXwGtjv56ECatJ1JO4LV8CrsBe8COG53fB7cZwMll8G1eksEq4F9xf3PW07COy3tpkN1obFIPTbPde2OmanwE1g2gQ4BraDb+AGeCBxt8f1MpCtUUnEAbj6dwPbHurCm1PWezPMAx3AMXsZCqaWBSup+oLcoJWdvwQ0shPNSX8uOOE6w+fwAYQBxptROBCCYV24v8NCVcn/ev07Cek6gB4qa0AL2ADehGxdRkRHWAoc0GfhF3ARpBXaMYLIZSC0o2eSyYUVFNoxkYifwI1pNaipHSRP0ZP4XEgrwytTYqs+zTiNcCk4Ic+Ea0C5GYY2ZiLq+LII+bST0g4PgX17AXYANQtsDD8byFJlKvw9fhepC0FlP9e+Kjrzal0jU+Fs7w9EmD+M2UD85XAxXA0XgboT0v0Ofp8/B+4AtSS0A8fDuePcn5S4zseopm2B1+ne4uA8XgYmw0swPPHjZH7U9bWeRBfifgddwPUwBrqB8zpoTTybwzhwzTqnPoWBEDZr9wbDr0C2nifC8rcED6c/wbXsXqhugZnBOerB/zmYfxvwAHkNXLPKPXUgeGAG3YjH9b0B2LZh8DGoQ8B255J7UFfoD3PBU7A3uK+qzWBnsC1XwVkQ9sx98I+FoOxwiN8Lj+Ni3x+HweB+Yh3Lw3bg/jQIQh/xTtH7+E6fEqoaH/dj9x+f8wDVlkHn4XFvnw16wVegbJ9jOxFOBPUIOFfkV1gBbNcA8NKwFNwE50BHUEeCdVvOY+Dc2BRmhrUgjOn++C1H2x0D7cH60/qRwKrQFxaGXWEx+A3swwkQ5JwI4eH4N4B+8C6sB+E8wlvPyuMjCw3qYJ+Zo0mtiNNYGuWMxG+n5wcH3OfOB2+HXhg+BeVAmvYo9E6FnTxXgcYzvR1slPhdSEGn4jF9CNye+B3sbDnI5nNCPA0jkvDZuMqJbbqToA+4QUyGD8EF7YA48e2ng2Xe7SDoZDzG3QC3Jf6BuKocXtSTQ9brxNEOZXAEOHldYMr6jSuF3WA8HAlKW7togrLDId6FfS84eQ+DEdAGOoFjtDssDteC45AtF8Urqcjw3B7E+dw1oN3URWB/1ocysK51QWl7n10d3BiV/X8HVoZ14BtYErT1iVAKO8I4GATKcb4SSmFP+BT6gPPTBbQvKNt0KXQF2zUJOkK9KY+1VG9taGoFT2ebXo09XdezNTW7NtH+XEa/Dm+ifZvmbuWzllrUVFurVq0GTZw48eSa8iRpM+HOCc+Ah1laXgTuhrbg5u2hsDuUw6vQGnqCB8RgaAlPgLdhLxY+9zTcD7bXw8VLiIfDn3Af+Mws4HMeLsq2/ASWrU6BmzO+f7/YntfAemaFr+FsOA/cFL6E+cD04fB44i6I6w3wFvCAsy22IbTDctSz4GFpO4rBdtwIqj1Yv3my9QYR78LGsBm0gUPBG7B6DraCrcE+XwLW7aG6KHjI2yaVHa6KrYofSuA4sO8D4WfwwuDzXn4s37E4BP6GtKyvFWhr5XO2b1vYBj6Bg8HnNgTLWRnK4FQwr7J9HuRexmaGp8D+T4D9wMP9MHgb7HdfsG0/wf/B7PAKfAWrw1hwrEfD3rAGDIGrQA2DXrAbPA/vw3AYD/WiPNZSvdTfFAudzjb1Auu+dxNMaor2bWJ9WoL+uE96dkRlWaBgaymf20dWG2IwWiDbAhcRcXR2ZHMJx7VU+JGONi28TWOJzdMC+aylkuZpotjraIFogWiBaIFogWiBfC3QMt8HYv5ogam0wIFT+Vx8LFogWiBaIFqgkVggftLQSAYiNiNaIFqg0VvA7zEsnrTyKNwtEn918UlyvTmPUvIctZTu96C2rSVPdcnLkRC+B1RdHr/r5PeL1AMwf8b375eVCPrF43zkd5vWzueBesjbgzKvScr1u1G7TkMd6/NseJN+K/4wj6ahyOnzaLw0TB+7x1qjBaIFZjwLXEeTOybNfgH3o8RfXXySXG9Ob0r2i8g16SUS/bL11Og7Hrq/lgfT5fslZA/7bLUjomd2ZC3h7UnfvZY89Z08JxX45X3lF8zfyvjyf3GM/NJ3i+TRu3DHJP4Zzgk3n8bScG91foxdX99iX5SyR8JSMABOBlVdfFVq/bwOpNhv4bEaiu9M2s5wfA15ako6n8Tz4JtqMnnb3QX87Ykd4FfItUn4JcbT4Qeoq9qT0f5V1PWBmC9aoJ4ssAjletjPB/4mzrtwDwSth2fVJOBvaj2T+D24PocNwIPX5837GQQtjydXvOnur74L7wSuhRvhT1gYFgMPWOv9GobARFC+w98UJsAd8CWomWAgLAT3Qb7yEwcPedvkurwF3A+tywPRdijbK/5WUZBt3Qms2/3kJrAv2bJf4VehB+P3N5my1Z2IzWESeICm7Ukw8+nJariO2TrwBMwLltsWXoMHIZc6ELkVzA5eem6A8bAS2N4ymBvuhXAJML993QL+guthNNSkLUl0Lrknmt/nWoM2tt2G74ZPYUNQtl+7pTUXAe3qBeVtGAbKeWV7V4FF4VV4CKa7SqZ7C/7dAI2n4etD21PoaTkKri4+R9aCRq1KaUsXtMRpK2xlHncx59I2RLpY66pWZPwIWtT1gZgvWqAeLbAsZXtIeDh6YTgOjgR1GHghLoex4GHcC9RVcC7MCS0hrd0IrJuOSPzpeA+N7WAErAlPgWujG9wJA2Ek7A9hb9oI/1DwkuHB8yz4Jkd5wHoAlcOF4CWiNu1MBstUl4CXk9HQCSzbNdobDoKgY/BoM+sdBOpScM/6EOz3DaDcszfO+KpeLsAZBZ3hOchu4zrEPQA/wjjQJtXtOyRlNB+vr8NCYNlnwKmQLS+Hr0AxfAnu7ReD8jC3rlmhHLx02B91HtwKn8L88AJkf2KyNXH9QWl77fUJOFccZ+X4aBvr9jLwMrSBbJ1LxKLQDuyXl4yv4HgwTXmReRRM+xluhDCOeBup8vk1DLrg5FgDHJylIWhJPPOAE2NLmBfSWoZAX5gDJoCGzNbqRGj8zWGlrMRWhFeD7aBrKs16F4ANkvjzcR0E87gJeJNzAuWKJzoj22SdToTWmZiqF+szbPx6YBvSWpGAfV0wHYl/brCvS8BVcDDUJOu3LLU4OKG1l2XbN1UKlpeW9poFekGYtI7BprAmOFaqLYTyXVxupj3BPs8GQT/i6RwCuNpuK1g4FZf2rkKgEtaGYBsn/zawHDQ75bmWmp19pqbDedjUef8JFCf1uD/9kIRNc69QprtH7GEA/QWrZnxVLy/grJuEfXd5UOLPFb8sad9DWGuW/Q70A/ckLxJBruenk8CruANCAq5r8kroAj9BK1ALwWRwXdekK0g8MskwGte9QZXAeLAc7fE1GDczeEgtAGvCG6BeBA8190wJ+8Zl+I8GNRbS9rIv7iXrw0ugngPfcQcdjmdwCKTc3fDfmoQt/+ZUmpeDv8D9K63FCLi/BrlHeYlQ/wfaMWh/PLcngZG464UEXMfCMegDb4M6B06FOUC7eSlQLcExcpz3hdag3H9/Bfdtx6wSQtoI/M4P5889EOT+br/mhf/BEAhyfz4pBArt5rGWMpOkEPXbyfdhP+gFT8OBoI6Ch+AM2B4+hVJQxt0FDpi3QCdtLj1ApOm94UIIk6kd/ndB468Mj8ERoJyMbgCDwDasAk6qtcGD0Qnk87niic78LOsTXCehk8F6wiE5DP/DsBOcDrZNtYB74Txwcb4MbkpqRfgQNoLrYH2oTd3IYF7lJL8fzoetwbYtDU7KuyHIOO0zAe6EjrAMvAVlsDfYlzbQFQZD0B54TgP7/DH4bFqOj2VeBKvBC7AFZGvDJEL7uAkdAM9Cb7gG7oDqxpqkqGiBglvgHUp041YfgXuWB8B74P7xBjjnu0MxBH0ePHm6rkvXu+tQWffbYLz6usrJvI7j1b1DdYH/gxEJtq1zgm2ZCOpb+Cnjy+9lVJK9AvdPaAm28xtYCzaHF+F7SMv9tB+MAfe/uSCX3FuCLHexEEhc+3cWhP65X9u/mqTNHL8g+/ArLBoiErc8iXsc9ws4BdL7jPYPcg6EsTDurZCAm52WSsqcXdpdO6hJcDo4zs6lG+EDeBm8JKTrJ/gvZffrB1K1e2mSa2Ti6vwBLVLh6eZ1whRCHkpXwzlJYQ7AAPBwURpxl4yv6gLhJeFB2A86ggOwArwG1cmJNhRagYt75cTvIJ0B6hXYA0I7PsW/Fai9YVW4ENYC9TXcANnxpp0LJ8JVBpDPDQLLV5eCh7WH7y+wIKwJ80MvcJO4CWyzh/1pcAJYnoPvJpWvRvFAOKRvw78RnA+zw3Kg3b3F26fJELQ2nlfhsCRiX9xZE3/acVH1TSJs7zGwZxLW6QeLw/LgpnMtuEDto+Egx0Pb7Q6zwZnQDdwoHD/baVnaLypaoCEs0C5Vybz4/4bf4Tm4H1xLbtr6iyEovY5CXF3csWSaJyuj4XDYuD/k0q9EbgMvJIlz47YE153+IPcQ11a+qq7e6yloO5gPXNfZ+pmIlWAB2ALuS/w4/5JtHJfE2N/R/0qtOuzdU55K4r18zJSVJzuYbUvzt4VfsjJ6nnjOyHuwCRwPQXMGD65zwD4FmfZTEjDt7ZCQ5f5G2LwlEPa8XfF7SXgEdoQn4HdwLGvSWBLTc8QxdZ6GflU3VjWVWe9pdrwQGk4hL4EHuweBB4YTPSg9AC5MB70neKMLi+h1/BqxOnk4qYng5WB5eB6eBg+lu+BUSNfrQTm1svxHUw8/jH/FVNiDT40HJ5K3ylXAyXwduAj3h4VhQbC8Z0G5EQ3Xk6dy2dGJOwR2AMdzexgMaQ0j0AO+Adv2KYQFgneKgo2NsK22OS371wbcVOzfoeACWxSqU1cSRsKIJIPj56JaIQlHJ1qgISzQm0r6gJfWQeB+4Tr0kPSAdl9aFdaC6g4xLxpLQPaFO1e8Zbo2dga1Prh+PFhq0p0kngjuI3PAveAh6J7nobITqMMg3Q7rmduEqZRvQtaDZeGBHGXcQdxAcA9x/dpn7ZetI4iwnWuDl4yHIC37dzzMCbOD58VekC3LXxB8EzYUdoTu4P4+CNx/bUtajqX7zDswG7j/psdSO7YH7XQI2OegY/C0AueA8+R+yCXLl4OTxA1xB4HtqoTn4Xc4EOyf9U8CbdUNtE2Q+/LW4Fno3n0cfAmfQ03SrmvUlKE+02xoITSQQm6Cz+A0OAGKIUiDBWlY5aRID6hx6WcMp5U29swk/AXbgpPZQTwDjoa0JqQDefod6HT7vBT8mSqjIuUPXvv5EpycMAi3E/wEtiVdXk19JWtO5bKjGYeAk8+F+hU4DmmNJNAZtoJR4MLtD9myj0HZ/TXe+r2Ihf7pLgbZi5eoKfKSkO63CZbt+EVFCzSUBT6koiPhUwgHinXvBefCS7ArHAseAOpZcP4Guc48LDaFj+FrULni3Ss8UHYG96cToC+4VsaAh15QOmz9tvVNeAOeg3PA/WgTsI2WtzB46IT2eeiVQrbsr2tevQDpdZcOjyXNflwAoUzjbIPyshLqvhn/9uCbpc9gJKjnwcvXJ3AmbAHufb+A/VEngf7X4G14HU6HbA0nwn3+MrCdh8NNYH2dwLKzdTERvqmxzGFwBXwLrUB9AF7CXk7cwbhBnkfa/ULYBr4D+29Z6gvQ7sq61wHDR4Hz4V04F54F2zs7XAB+klAJ9sOzqgNY/zjQtvvBtWD5PcCylJeHcj2J0uEViVsjJDQqN48vR1xNwzWYKoZb4GkD6DrYP+OregnhtgRHQ/ckbQNcjdsuCacdB2/PJKI97o9QCk4SCboej4tf2aaDMr6qlz1wHDS1FoTJUF38UPI48VULuAv+ZwC5EEr1JArhzQiPgDmT+PVwnYg+PxjOBuWEcvIfbAA5sebK+P79sjrB95IoJ6AbSlB2WHu/BbuGDLhuasvAYXAJBN2Jx7pXBReS0o5uULa1BMxzEijt3RnWB8ucG1QfsB+tDKRkGZUwG8wCjrN5lX0dBSFsXJNXHmupyduiUB3Mw6ZuxE8Wqt5GWo6H3QKNtG2NoVn/RyOOq6YhI4lfqpq0ZhGdx1qacqOeVsNcTwF3Q1eYEzy4PfC8QFQnb6m7wSPgLWoy/ArVaWcSdoFF4XgohyFwHywBc8AL0BE8tLLl7fYi8DLgARlUXfxBZBgGG0EbeB9Og5pkW7wBfgTe8G2LfbRvR8IDYH2W50EcdD6eMeBBPrVyDC4FD/ts3USEm6YXpUqYANoue6H8Rdzb4CXAMTkb0nqUgM950fgYFoO9YCKkZX9fgXLoBTuAF8lyKIXL4RmIihaIFph2CxRThHvV99NeVCwhWmAaLJDP7YNqWkNp4uLUWR6gHWrJPZb0hRN895rWTARKYeZ0ZDV+65JsVRdvPuudL/uBWsJtSS8F37Fny0vPrFmRpYRPyoqrj+AiFLpgLQV7+aotj5+UlEKu/hE9RX6qENQSTyn4bLNTnmup2dlnajqch039tGv+qakjPtNkLDAXPXFvy6X2RGZ/WporX5ONy2Mt1WyDghVUczV1SQ2XhrrknRHzbEmj40eLM+LI1bHNjWgt1bHFjT9btGnjH6PYwhnDAvmsJd/9zQjal0b+MiM0dCrbeNdUPhcfixaIFogWiBaIFmgwC8wolwZ/Hh4VLRAtEC0wNRbwY+m+WQ/+Tfg5+CYrvlDB1SjINzp+Eboh1Z3K+sE78B5sC6PAtvg9ox8gl7TR+nBHrsQ6xnme+J0pv9M0vVWI/tRXH5am4Hbgd/C02V7gj3Ifh2IwvtGqtp9JN9qGx4ZFC0QLRAvU0QJ+L+laKE3ogrsdfAKrQH1oMwrtVR8F11Lmo6T783u/1OybLfs6DvaBmr475qF1OEyLXubheaelgAI+uxBlXVDA8gpZlPPC+aH2BD9JnwjLQojHOwMqn59zFLh7fnEp/IzfX+9rW6Dy/RJifXy60oJyS7OwrvqUG8Oc9VlBNWX7RdS1oTv45dcyWBEcL8etOmmj+rZJqLszHttou7IvxtqtNAdumuFLtbnSfW6qNR3X0lS3ubE/mIdNfWf3d47++Fs8N2bFe8HIdfD5Bd5pmb/uO34ROagVnlLI/lK06R7u4deaDedSrueN851++FL4t/iXyPUwce6ppZC9PoiaIr8A7uGbS7YvO+134sK+nf1MaG+uL0IvSOZScI/Ilusy/SVWxyCXzcyT/sL6UoTtf1qmO761yX21FHK1xzLS7SGYkW0qBfeQbJnfPubSGUSeniuBOMemFAp1/lFUbuWxlnIXEGILVlAosO7upmR9Jsl+Ne5hdX/0XzldPLelYsbg75gKF8rrZuBitc1PJ3yHOwI6Q33IW/SJ9VFwLWU+S/qL4O34ZvBj0BPgedgIqpOLdVx1iXWMP598nWrIa5pj8CXcCu/Dx7AYBN2D5yMI4xTcPYlbJRX/Nf5RqfC2+Kda03EtTXWbG/uDedi0ukvD9fTxqqSfbuqvJjg/bgcPOrUzlMOj8AT8DB5+/hjgSQgybB51LRyS8VX9hcBH8P8Ba8IGMBKce84x555y/lr3s/AJ3ALFkK31iUg/v3eSwborYThY9t/wMqwDrtPeoI4Gn3etfAE9oAu4Z6k2cD+8B2+A7QmX5rfx3wiW9xU8ACVwFkyCF8CDPa21CbgmbZPPHAfK31p4E2zjK/A9dAf1Ori/OCY/wf/gMbB87d8LlO27D56DT+EKUOlLw8yEh8IHYLkvgZeRXDqDyI9hODg2q4KyHdrEfc56roKgvfGkx8PxUbPAveCY2s/HwcuF88L5sS1oj3I4F0I83sy/BfgM1/5qu6Og3pTHWqq5DQUrqOZqcqWmLw250usa5zuG8XXNPA35FuFZF2vrVBkupFvhhlRcU/BqzwWTjri4Vm7ATn1LXZ2rqc+fYZbDSaDtg1zcD4YArhtBWSpcnfchElzEBdF0XEsFaX9jLCQPmy5N+yfD9Smewj8SFgd1J5yc8VXNn2H4DwIPy1+gE6h+4FrP59LgmtkHWoLzdDT0BDUf/AAdwcP8ElCt4GpYwEBKHjo+v0IS5x7n84vBTGDbdNV3EPr3Dv7e4KHsOvI5tRt4gHUB86vTwb0ryAPt0iTgpeHMxB/a0iMJ/46b3V6T7gDrUQvDdRlf1R/9G5T4dezvGUnYw9161epgv1YxgLxABDu9gf9CI5GXg3dhA1gK7Kc6Ee6CYgPI/l2T8f37xX3jL3DM1RawZ8ZXdWm4OPFbj/bcGKznJwj9Xha/42Geo2EoWK66CXaAQ0CbK215asb373j7tWMSb3u8qMydhAvu5LGWpnRmWhvhYBwPH8JX8DyEG6M3JNNeg49BAwUjupkfAw60k3FDyNbhRAxIIrvgPgYa0DrCZC1LwtbtrW4/UBdBa3gaXIT3QRjcnfG/DyPhdnDxKgdUXoZRcAM4AfJVBQ9oj3bJgy1xzwZvj9rhWAiyH/bH9tjmm8EJsjxcDkGGw2LZH799UE7MQWD/14IOcD98DtrdONUGhoD12zZtn0uLEqmtwvPrJJnuwdWed4I2NZ/t2QwuhlVAlcFL4Dg5xgvBPPAwBB2Kx7HSHhfATKDsv+P3HtgfF7g6CeaGG2FJyNbBRNivk0HbBznfbkkCtsMLjxtSbVqRDK/WlimmzzAWcE4Mh2egPcwGXcE5rpzjzg3n2QnQEtaEleET+BLUMBiX8eX38hDZJ0FPsOxNwbr2AcvrDc/BLmBeD6sT4XtIawUC7mUbg8/vC7+Dz9dFfcj0FPyUZPbw2i3xB2ddPK5Hy5d5QVsEPZp4/sD9GmYNCdW4TxDvGncdrg0HgboE3F/t8/9gDUjvtY8TVl/BX/CyAfQNuJcF3Zh4zKPtypJwcOyPNtee9sdxLoOFIfRRtwU8DR7YtvcXuBqCbko81vMAlMFa8CvsBZaxOcwC3aAP3AkVoHaAUEYmopqX2YlfFjyX1BhYAkYbmN4qKVADNNQA6AWl8Bo4QMoNfjvYGFaC9UHjKSfJotADdgcHfz5Iy+cXAdvqQXYbaEAPFwfEAbobjoUOsDM44E6qA+FvcMJPhNXB/IZPgy2hE3wNd4CybNuyFTjwLnLz1UU7ksn65WQ4FK4AdQIsDfbVjUh77AJO1GFgm93ERoD2ag1eOJaHIMO2Ry0OHTK+qo/QtONG8BbcCy448+wJt4ALZFtwQtqOPmAfO0JaxQRsz1Pg83vATdAe+oN2dLzXhK/B8u+D5WAumAccl6NBW74K2sD+ON5qe9BWvcG2LAAng7KcTcCxKoN9wTjTXTQ+9wlky75dnR1JeBTcnMSviDsODoOTUvhsWh0JzAnaMqppWKCCbgyBwbAh/AVXQpCHlWuvPME5fBm4PidDUCWev0MAtzjlb5XyZ3vHJxGuAw/b8hSn4H8FXgDXzP3g/vAphPWON6Ncz59KSjhQk2zVOj7vGg6yf+6taZnHtV2e4F7geglKP6890jYIedLuVQTc896DveFjcK/YH4bDkvA6WE9ajlFQdp0hXjc9Ho6zfUrL/nwD5QnP4B4HM0FpCvvh3rMnqCHg3heUXY92s+yxUJ7iAPyjwLR0u2uaH2SdIp/TrpOmxPzzo7JU1PTxZk+WqW2Fh8wq4ERYH9rBHBB0OZ4f4Te4AvpCkBNeAzlpXDjrQi51JdID/7ok8Q7cMvgTFgEnZC9wkTn4s0F16k/C9eDkdWCOh1VhAVCW7aA7GV6E7IsMUTnVm9gyGAT9YA14ENS28AysBB6Cz4OXEf32/y5QF4L15is3uA9gIVgStEcZOA5fgBvld9AHToTFQVuNgLS6EFga3oEymBM+h42gLlqHTB629lWdAbtmfP+8aIvnYBlYDbTFFhB0DZ5fwbZ9BLXZv4Q8ncD+qx6gTQMXG4lWBMc8W87LtByj92F8OjL6m4wFXPMDwU/IXIPqNfgDPCjEObcEvAHO0zAH3YfmBuXeE+INr+BLLXqbdNeU8996boFNwYPCNzmuH/dL19vTsDKk5bp0TbtmwvOb4PfSUxe9SSb3unCAbYDffTctbdECgi0m4F8+naEavwe2azFbrj/7fCZYt/m07U5wdIL7n2vYevNVWfKAda8NLybh4LyOx/6G/jjOK4H7yy4pZsV/PzwDB4PnRBkE9Uk86Xos2z33brD8h2AH8IKhrcMzeDNvDE/WU4t+Jv0b8PxQzo0vwXk43dWyQC3woLkHxoET7i9I66dUwMPASR+UndaWhOxN3LxzwS96UvoWv5PsSlgdHMAXwElZDJWQS07gz1MJLv7xMFsSZz+CJuOxrLpoDzI5WezfS+DkcVEo2+8k8EAOehmPeceGCFzbPjoVTnudrNVpTJLghjYJXJBBn+LRVk7obWA7cJLb5/XBS0VQruc/I9Hn66J2ZEqPkzdtF8HCqYe1hbaePRX3WMqfbf9UUk6vdpEw3m6sYczcEJ2TykvDEHBTrknmC8/UlC+mzbgWcBMeBB5oT8Ch4IHh2HvAeFivCd/BaeBB9CQsD2GeGeccfhjcQ9xXnO816XsST4RnwPq8aHwN74F72SPQG2yDe8WekNYPBI4Hn78PbI9tfBfqcuA+TT4PM9vufumbiV0grVMIPAWW/yNsDP2gNmnTB8H95aNUZve5O8EzYjEoB+t+AAZBZ9DerWFByFeH8cBysARon3thSQhy/Oy3bfsWNoGtIFvuW3/Ac/AqrAUXQtAheHqA7XW/HQoV4L5q/uGwNri3Wta58GwS9lzoDn1gJ6hNB5HhdrCOnuBYfACNW3l8OcJD+9JUbw7H7wJT18GpGV/Vi/6LkvBvuMsm/mJcjaJRNwUXhboanBSLgvnbgJoFnPgbgAvRCacWgkpYAOYBF3OQA90RToTBENQJj5PFMi8H6wvKDof4tLsIAesMbTDNxTwBehtALpz04lyP8FbQHn6FWUHNAX+B/SiDDyFobzz2WV0A9kO5UOyD8nB2gs5rINGBuN1gI7Bepb2diEcbSMlF6/Pzp+IOwO+EV7Zt7oyv6uPT5RL/87iWvz68l8TpLAH3g+0aB+oGODPjq3rRVvZNuWAtIygd/oZIF2wuvUDkEVkJ2tT2dgH76/ivALXpWTLsUVumfNLzWEv5FNus8+ZhU/eMsA7TNmtJoAzCXJ8Nv3N4HUivZYKZeey+5MWgAmYH5TObQU8wLqyTJfG7ttUa0Crj++elFK/POffTmo/A5mAb3I+qUykJ2c87x8tAV60KoQznvXtLkOvW/iyQRGgj8wfZ/zVhQ7DPQZbTNgRw0+F2hMtAN1uLEeF+twq0SCXaDvvh/mC5pivtEtprW1Y3MpF5ta96A9YC97WVIMj+9AoB3JnAs8XxnQuqk7aznAHg3hX0Kp51E1YOkSl3Gfz2w70mLdu+NvisbVDOi9D+Tvg7GonS8Ybdix2jHgbqU3mspZqbkUdBh1DSu+ANTv9H8Dqo62AMHAXHwLfgBFK/wZuwG9wMT4GDpqGeAXU1HJbxVX3n4TH8O8NQuBY6gOV46Dgpvaz8Cg6KA+YBeDa4eGyHA+RgfA3nwB7wHpwI6nII9WWH7cOBRmYp16XBLOfCJ2A7Nobv4WCwjB/AjUFdAU+D/XocJoGXBie3fTkFDgJt9SKo6i4Nplney7ALnA9fggvf+uz3PqC9RsHKkK3LiHgFdoXzYAS0A/UXzJ3x5b40tCDNZ4fAwMR/OK4LfRyo7vAjnAB7wheJi1PjpcFxGgylkK0ViRgNl8DuYLu1t/Yqgc5QCafDSVl0JRxk+21nQRdqHmsptCO6tVhgOtq0gqbNXkvzYnLDWOANqundAFV5aShrgHqmSxX5rKWWBWrhRZTzMywP5eBtcXMoBuXhPAFmhl7wFQR5oKwGz4IHjRv723AmqOvBg1MNhG1gcbgDbgcX8JqwCbiQd4EuYH1eGDysPRhbwX7wE3go9IQdwcP5EHgC1A3gJSQoHX6ByIkhIeWOwT8QstM8nDzoFoAHYX2wncWwLrwLal+wX53BZ8KN2XK1V39Qm4EHrroF7J86EDyEg7wU9INu8BWsAGPBPm4La8Fk2BS0dba0U1+wrlHg87+A2gO0nzoaLF+dDvbVci1f23aAE+FRaAN7gbLfjomXvAXBMXsW1BnwfsZX9ZIOb0+U/cql14hcBraGheFb2AjeBPUnDNSTQ87dIOeo/f8gREQ3WiDLAkMIZ6/1rCwx2EAWuJ96fHNQ33qAChqinvruR/2Wn8/to4aWXEfa/tWkezh7aET92wJekrzMRDURCxRoLTURaxSmG9GmhbFjLCVaIJ+1VNIA5vqEOr6ppp7niP+7mrTmHP18tEtzHv7Y92iBaIFogRnQAvncPmbA7sUmRws0mAXiWiq8qaNNC2/TrBIXIeyPfKOauAXyWUsN8UlDEzd37F60QLRAtEC1FiglZWCSuh7uy4l/ejmDqdjvUNWmO8nwFvidpB9gCSiUxlCQF5Js+Z2BJbMj8wivTd41kvzn4tr2QsrvSfm9s0KrnAIL+sXrQjcwXV68NKStEf3RAtEC0QKFtYBffO6dFOmXdetyYBe2BVNXml+E9ovUR4BfPh4FjV0H0UC/BB1VjxZobJeGx+nrbNX0dyfin4ZZs9L9DZB74Pgk3t+vDb95kURlnKG8Lp1E3IbbM/EHZxAey89FO+L9Rr2TMmh+POeAvwkwAh4CfyMgqDWepyBXed6yB1WTZn7rawzqRSPCx5NX4V+uMTQqtiFaYCos4MF9F/gbRP7Wj/uG2h38LR5/E8s1vCUEzYHHNe6avAYWBdUDDoAr4BZoAV4O3GPMa7xfZDb/DuC7X/N7oG0Myr3XC8QjcB+EeLyZv3mzFq7f2DetDyjrOQp8ZhisD7XJeg4F+30hpPdX17Z7pW2+HjqCuhS0j313z9wa2kIXcA+0PPe2ayH9RfbtCGtD2705BM2H5xJ4Evztq5rkpwWW4XhoZ3UIGB/UDc8xIZC4G+EuC3vDmkncvLiOhf3TbuG8a4P/5CT+RtzqPkWZmzRtZl8dq7R85gaw7FNgFpgZnCeWH7QXntD2/vi1jf3TprnUncibwHJPgzBe/fA7l66DR8F+BnnWHA+20/m4DKj2YLy/2eZ8mRPqV/n8nKNALZlAOe2qKcsF/zf0zEp3MY6HI5P4wbgaKS0n9iRwkWi4yeCESms5AmXg4q8EB6kMeoNKT/gOhL+Gy8GF5ORyMY2DxUCtDD9DWQ5aEFdbfWSZ7vqNFmg7tSeETSUTEV/qboHpsJbq3rgZNGceNl2eLvoR+xbggeOh6yVYDQbXsuvcNTsC1gX1LJwHrmkPjS/Ag8ED/k8YBB6Uq4Jl9AIPkzvgRnDDvxA8JNzI14OXQZ0A+lcA6xsF7j3qF/CNkHvEfjAWZoKd4THoDGuB8bW9s7Ye++FhtBtMgHAA+ibNw3xx2APKwTavDu6BG0A70Hb2qwz+guOgK9jPW0HtCu+A/dEen8BGoF6Fs2BpuAYqYBHI1vdE2FbHaCewXvfWPeFhCPJSc3gIJK7lvQCnwEJwLoyGvuC4fgFbgxoKtkM77gjafi7I1nNE/B/Y7svAdvvMfPAd2EbtciXcAUqbbpfxVY3Zj/g9L7aEj2EVWBHehQGgyqEH2Afz7w5LwRB4FJSXHMd7U9A+b4L51E0JtsU+fgMLgPn+BMd4D6hWeaylasvIJORZUCceOgxOAid3K1CrwWLgwJvmwgoyjxP5RHBSO6GdpLn0FZGvwLapRPMa/x6smcR/gLt54g/OZnjeTwLr4H4ZEnK4fYhzQqRVQuBXWDaJfATXyZStq4nYJoncH9fNojblqi/7mU2I0HZuIHMkibrGu8BNc/K3gKCOeI6EY6FriMTVNk4mn9HmLcEJbfhocLIq7eSEOwisy8nqRFSzwV7gM/2hGJT1WLYT1zSfCXLxHwDGOx7hGbxNX3mupaZvkAL0MA+bummelqpyQfzuNbPCYHCdBB2I50ZwrbtJu45KE97Ede5uDJ9AkHN7ySTgGjkVnkzC++Nel/jXww2XhpH4V0ridfaGh5PwL7jdE7/OOFgI3EM/hH7gGrT9telzMqT33KGE94UuMB4Wg9KEl3Bdu6oSrEN5eHsglUF671yX8KugtI19KE04Bte6lgafLwY1M/wBHpDZ+p6IsMeadh/sBm3B/Xd+mAnMF/YivFNk/m2S0Lm4F05Jqdqv3Xvaw0ToAqUJT+DuCml1IuA4tEgiPaucD53BfcxnShMce8ucC7aHh0D1hcczvqKi4biHQ2mCZYQ5Uo6/BxwBN0JQazy/gXWeDJdC0OZ4ngXn3mRwvpQm3I9r+d3AueMeX6PyWEu1F1ZjTf8kOqGeg7NhFBwEXUEj7AZ94CZwsmhEw2/AndAC7oXLobrOOUFcNNdAWJx4M5cNy9gPLG92cDK8BmmtSCDEuVCDP50n+HOlW6dtex8WgzLYFrK1RyrCcr6A0lSc3h9hfCou3bZU9BSvk39NuAKWBxepZS8I2tSL1PWwC6wGe4OT5RH4P3DSPQg7wTNwKlTA8+AkvwpcjHeAk9PySyFbg4hw0rt4X4C3wUlr3AawJ2wEjv3d8DFcDYfArfAYPAyfg4tXm54DUdEC9W0B9493UpV8h9814PxX5ZnXqpdvcVxb7aEEroMg536QZQQV4zkZesNI+BmMq0nW8VUqQzn++VLhMSn/RPy25Vqwza6f28DDwf013S6C/5L7pn0KKk88C+NqA/fUoL/weADVpNGpRNsV+ml5tmXrVLp7gPVrq8ok3jq+T/y5nK9Tkd/gd29yD7sP3HPL4Q2oqQySM0rb8G9iWoDjOgkuh7TsS1q228tOsIfptkdZxpLgvhvkfurl5h64EGz3jjAYlM9sBxsbSORcSct5mp4TtlnbWZYqz7xWvYR5ajtto3WmpZ3VT2B/C6aWBSrJCb0XDEuVt3XKPxS/E115SHroeXCuBnbaAXkU0gYjOEU+44H9FuyQxHbG9SbnIbUhOLH6gMZMLxKCmTpdYMqyHODqZLoHZ1rGWbfGXxnehV9AeWAaF3Q1Huv3GSfRJpDWlgSc9EHaorpLTDvS9oMO8CNYtv3YFR6GVrAV2PeHwHqPg2PgEvDCoWzrsfCMAeQECxO+C/7bwTzOh51gMXgCHJc74FcI2gKP9e2cRNyJ62I6KwlrpwMTv4tgFbD8peFEsN1PwhwQFS3QEBZw4+2UqmjhxO+GqhapcjKv5isH19IEWA9cB2pTcK0uD5UQ5HpzLS4K7hG7w/ZQkyy/I/yQZLJe11FNWo5E99j/gX3wgLKey6A6uae6f4S9tT3+z0CbeGlw//JwUhvBOxlf/i+WZ7vuTh5dCnducN+yTg9sD7eZYF6oTtrQS5fSJi9kfFWXt9Pw248hSVwupzgVmR6jEG07S8CxHJdEOsafJP7gjMSzINhe54Htdz9TlqGdwt7u2A8Ax28i3AXOgdVhB1A+cx1cbwAtDul5Z5x5eupJNDOu5+PXSTidX9uUw/dgfzxXRoNaC7RTG8hlA6KnXlZWCH1OIaXwCDghj4N02XYg6Hc8DsAS8CloZDUKwmTJRKRePIBdrB/Dkkn8ObgnQTcwTa0Ar2d8/7xYlwfXS0lUKOufHP/25UpPH+zzkH1M6pHx+MvBhXci2D8PRPvnIu+YxRuE07K+V9MRKb/POilceEE+3zkJaGsPcKX7LSwGS8PBMCJBWwW74c2Mka56GS6AN+FtcKGnx47gv2Td5g2yXsffRaBGVjmZVxel9ndz2geuACf2GWBcVLRAQ1jAzXoP2B7clN24B8NfoA6F9WBjOAiuAi+/7k+XQ1fYF64BD75s/UGEa74UyuBwmAWUa6A7rG0gpSvxXwqrgvW6ZxpXk9xPbgDfpCwKc8EnoOxTqZ4sWcf54B7mQWZd6kNwL7kauoH2GQJTq8t48Eywn6vBPeCeow3dLy4Gba/bBqrTWSRoL8ekC9wLajjMD6vDfZBL2np9cP+rTl+R8Dhorx6wE9wC7lNpuY89A/bLdrtHOsbqNnAM3GOdGxeC+9tEUJZ9LAyFP0FZziDYAHz2TrD+tG4lsC4415YF59vzUA5qIGwFveE0uBx+grthMPjM1nAHtIZsbUnEftmR+YZb5vtANfmPIn5T2Bveh21hZwiqDJ6U+wv+uVJhB23WVDjtXYnAXTACFgQn5UJwIwyDx0AZ933G98/LOnidTG/DwrAAOIlzaT4iO8BrWYkrEr4giSvHXQKKwX49m9Af92Pw0rAWOOl+hJoU6su+6IRnfsPTDkqgIon0UNd2ao4qJ/Nqe0wbDbbhHLgZlBuY5QSFjW92IlwYh8Nh8DN8BTXJNnXKymC9Y5O40M50Ftv/FAwGNwQXhRN9KYiKFqhvC7xFBe5PB8A2cD+4iQd5aOwErWA7eBXUhnAEeCh4OPeGn+BreASCzsJzHLiJvwP94GBwT7sXeoKHnXP+QVBngPuDz7l2rP9JULbnj4yv6iWEPUTawIkwAY6G8EwX/HNBOaR1CYE/wWc+hYPgC1CbgWv//8C4MvgG1GCYqAfdBr+C6/gBCHKvDeGr8JvnQPC5U8F2qy3gGLDP7uPuIen+Eczodl6fgNNB22jv30G512rLWeAvyCX3vP1hKXBPtd9Bb+CZlAQG4B4G54L7nWfEl5AtzzFtbLut+2yw7T/AquDc2AAse1MIegWPY3VtiMD1QmCf94IKuBiuA6VNxoC2t8+Wa3nPw/kQZD7bOj+YZxioneBQOAssYyP4ENrD3dBwyuPLEZfSqmCAdvifAw8jZfz+GV/VSwjPRLActqqKznTaieHz2fqZiO5J5Pu4L8BqSfg73BUTf39cwyslYQf2S9gjCffF/QxKs7AtamMwPS3TnKSLJ5Gtcd1AzoO2MBusD6/C9aCcaI9AaRYdCKeVq750ejGBd8AFp7qCE9b+uUlor21AHQivZXxVG9ab+OcB238LOInVu9Ar46u6gLmQFknCLhLL7J2Etfta0AregDVhSRgNwcZ74R8BLeFIuASCQtjLjRvjckmCY/5e4u+B60Jo0spjLTVpOxSycwWy6WDatG8h2zWdyvINlOusqWpmOvY2hD2kqfazun6dTIKXgnpRPmvJjb4Q8oZ3A3jA/QhXwiHgTduDJ9wW8U4JT8C/GXg7Px/uAydFBaTlbckyPkgivTB449VdAMaBB6G6B7yFecPrCJ+Dxh4CqhPY56cNpLQyftu9GDyaitfbGT4Fy1J/w9rwP/AC42Q2/Ta4GpSHsIdrdj3fELc6BNme7PpCmq4HeD/wIN4fbKPuq9AlCWtD7f8hhAvYRfjbgTdex+AZOBjUt/BXxld1wToO/1MwBp6Fq6A08dsfbTkAwnOf4N8FrgBtbx21pAAAQABJREFUrQ02gEnwK1hOUAjr7go3ge3ykrETqD6wLDxhICpaIFpgqizgBd111hS1BZ1yL3J/f6spdrDJ9Cmf20eT6fSM0xEvDaNmnOY275bGtVT48Y82LbxNG3GJvkGMqicL5LOWSuqpDbHYaIFogWiBaIFogUJZ4PtCFRTLmTYLxEvDtNlvej49ksr9sUFUtEC0QLRAtEC0QINYIF4aGsTM9VLJeEp9qV5KjoVGC0QLRAtEC0QL5LBAvDTkMEqMihaIFogWiBaIFogW+K8F4qXhvzaJMdEC0QLRAtEC0QLRAjksEC8NOYwSo6IFogWiBaIFogWiBf5rgXhp+K9NYky0QLRAtEC0QLRAtEAOC8RLQw6jxKhogWiBaIFogWiBaIH/WiBeGv5rkxgTLRAtEC0QLRAtEC2QwwLx0pDDKDEqWiBaIFogWiBaIFrgvxaIl4b/2iTGRAtEC0QLRAtEC0QL5LBAvDTkMEqMihaIFogWiBaIFogW+K8F4qXhvzaJMdEC0QLRAtEC0QLRAjksEC8NOYwSo6IFogWiBaIFogWiBf5rgXhp+K9NYky0QLRAtEC0QLRAtEAOC8RLQw6jxKhogWiBaIFogWiBaIH/WiBeGv5rkxgTLRAtEC0QLRAtEC2QwwLx0pDDKDEqWiBaIFqgwBaYhfK2LHCZsbhogQa3QLw0NLjJY4XRAtEC09kCvan/gTza0COPvNVl3YmEd6pLbID4banjinqqpxD2qaemxWILbYGmdGkoxjirF9pAsbxogWiBJmeBZ+nRa3n0qm8eeXNlbUFkR/gsV2IDxd1KPd/VU13Tap96alYstj4s0LI+CqXMMtgK9oO6aF4y/VSXjDXk2YS0X3Ok70Xc/OAF6R54F6ZFc/PwL1AxLYU00LMzUY8fi+aySwM1IVYTLdAoLNCVVmwPrlv3Gg/yoOXwbA6mtYJL4XtQO0KZnkTf4l6d+N1TjgXL0u9FJNcnGO6Fd0FavQhsA+PgU2gPr8ATUAYbw3kwEGaDZ+BxcC/bByrBeofBmzAXHAovQm9Qf8P/wVgDyP3gKJgZfPZ0ME9tmpUMR4B12s8xcCGoHaBMTyIvJleFAK4XitVgMmi7i8FylO2dCI/CbqD9zwLbuxTsAtZnnougtkvPkuTZEx6BDeAP+A0ugEmwFnhOXA6fwRqwBVwJH0HUtFqgTZs2YXCnpqiT8ngon7zVFXtujoRexPXLET8tUU5uF/iMoO400kUbNZ0tMI1raTq3vnFWn4dNPXTPAQ8g1QXeyfj+++LBemJWdF33p1N4LtSRLiJ7b5qPxLNTGTbEPyQV1jsQbgI/oVBzVjn/eT0tFeNlYSh48VHzgBePoGF4FkoCi+PuHhJqcb0Y9EjlOTDl11udfcpI801b0Mp4DgmBxL0Z93hoCbOA9m8LHvRebFQb8LJRbKAWHUD6qak87oGHpcJpu1tfdW1PPdL0vXmspcxAFcoiDvS+oPt0VqHzE/ZW7cT/Ga6BcMPdCX8ZBHnD9+YX5E2wK3hTvRty3QhXJ/55SGszAtbpTdOJ8yXcCMrJOBA6wZ/gO4tfQDlBfW4RMG0w/AhqHbDcpcFyJ4Ebgn3ZEpyE1jEXOHmfgeGgtgPLPBt2hEXhEhgLbjQDYTHIbg9RNWoZUrWR9vH2PBxs71KwD7SDsOCvwz8KlBuLC3phMP9lEMakDH8fcKEuD2vADfA5qI1gNaiEwRDi8Varw0l5FUrBTXsMXA6+G3BeHASW4yYyOxwM5aA9o6IFpsUCa/Pw7VCRFPIx7vDEr9MZ9gTXnmvA+VgXtSCT83Re+B1WBeNCPXiL1ofH9KTUG/9tqfDD+N1bsnUyESOSyLGJa9tcK+411rk4pDWEwMQkwr12PBSDa/Ut+BbUSLDdddGDZHL9bgra6C6oi9Yj07GpjK/g75sK6y2H0/Qg91O1LswBxxtI1AG3I3wZImpwr02lvYt/QCo8HH8feAYGwmCIKpQF8rh9uFA8AL1Bqy3g5Yyv6mUhHA8vpf+IjO+fl5P+8f7L52RfOonxYPWANi5bZxORK95J3jM7M+EzoEcSb5s9vFonYQ+sRRO/C9O8ae1GoH06IuU/LuXXm92vU4k7D1YAF/+SoE6HZTO+qoWcbk8SXa1zMSktk9QDcD3Mg+xj9iINaRfiWSoJuPFcFhISd/skzrG0fNusdgXj1Myg7TsaqEWtSP8UVknyLYJ7ReLXOQzmT4X3xF+XclOPNF5vHmup8XaikbUsD5uuSdPXyWr+kCRcjHsBzJRKz1632eGQ9WA8XoCDPCCd52m5Z2VrZSJ8YxI0N57BIZC4A3E7ZMUZdD/yTUlQum1eRo4JCYnrOg9K57W/2XlDvmzXPTtoNjwXQdgvjU+XazjoFDzuEWn5iU9auZ5diQxbpTPl4XcP3DCV3/HIrvNM4mz/yal8zdqbx1rK+VHa1BivDw/dAz8mD9+N+0Hi1xkL/cEJ4mQIlwu8NaqSVBfICeCC7ADpxU2wyHfaH4F56yLLmxe8xJSBlxLbui6ocbA82FYn4IJQKHm5csK+DtrkE7Ad2mNOKAP78z6sB3XRaDIdDh7kd8ELUJs6k8E6tJvyHf4z0NNASrfhdyx9B2CblZ86WGcZeAF4CnaAuuhhMr2cZByF+zT4KZC6BnbP+KrGuD3+EUk4OtEC02KBZ3l4cwiHn/PVi7LrzkvDJPDduW8SDoVSmB2C/sTjfFQLQ7iYe/D8aiRaHVyD6f1iRcJh3eCdolfweWH3DcimMAh+giAvEfOAdZZCWwiyveOhBLYG22L+IPezI8FLknvNg6DcX2Q2A8i+u/d4qNam08jQLclUgeteOzkJ69ie8EZLG2sLdT3YhjZguw8G13xQKR7bpCvmUa+Ce0vYGxyXPcD2Bs2P5zfwXMjWhkTsDhvBBXA1pOV8uBJuSEdGf90s0LJu2WrN5aB6kKTlRAo6Hc+1CU7ao0JCLe4A0r2png1/w1mQre2JGJQdWUPYCe+CS8sD9OMkwrY5aU8BF5QLplCyDz9nFWZ7vEyk5SUmtCcdn8s/iEjHcUnYHYbDc1CTtOm4rAx/EHYc0/o2HUj82bb7i3gvF3XRrFmZDE9I4tx8/wQ3s80gLmiMEFUQC3jAHQ27ggfP43AYeLjdA4PhGHB9Gh4JXeA1UJeAz3rAfQ93gboIPPi9YLwHu4Dv9m8CtRVYbi6dROTS0BY8TI+HIMvwErF4wqe4L4HyEDwA3JvsxxmwEjwM6jr4DHzW/XIsqO7wNiwBb8LK4D6zAIyCmnQuicvCpuDB/n8wCYK0j3aYE36AO0GNgPPgQHDPuQ+sW80PfcA26aoHIeyPR+LfFvrDeHgAfoGgMXjOhydCRModhN/y5wLH2T0qLZ/ZGL5IR0Z/ASyQx0cWHjZOpJZJtd1w30n8JbinJn6dtcBJltaJBFqnIhZO/C4sF4dysT8F6XyLEj4IqpOTvGeORC8hPhvUHk+nJHB6iMTtDPemwnq3hh6puNBWo2xv0DZ40mHjs8Mhr4u7uvaEPLncPkTumEqwD3tnhV3MQb4LcNHLFTAHqNnBcLC1cduDG0+29iGidypyVvzLp8J6j4BbsuIs2w1iB7A+2+5mlFZbAifAaenIpuDPYy01he42SB9mAJt6KaiL3C/3rUvGWvK4LtesJU9TTz6ADnpZqEk7k+ibrKjEAvmsJQ+PamVB48ePrzFP6mEP2F3A27q3zd/Bm+el0Be6g2W9m/hfxH0clAevh5E3whZwH7wF6XjL/QY6wJmgTgJvm9aVLdviu4VJ4LtZy7NcNRMcCLOAh9kYuBL+hBVgc/CZH8FLjzfdIaC8tBwFFWBb3wBvwaoHbJXxVV1wNsV/Blj+0TAPjAUV+qi/pvaYXp081G3rbGA/K8ELkf6g/fB44bIfvku6GsxnW1xg5rV+x8l0tTu4qLT5RHgdHoSggXgcB+Xz18F3BhL5LmE52DZE4GrnI+ApWBs+gaHgu8C0vEhcAx+nI2d0f55raUbvboO0fwa3qevRPcF14bq7ClyXUyvXuOtOvQO3ZXzN68VLmm+SxoH7zDOQVhmBPqCt3H8uhygskM9a8hCvVvkUVG0h9ZuwAcU/Ur9VxNILZIFwafASVZ28hA0CP21oUpoB1tIMZ+9o04IMmetyuxpK8vLhm4eoJmyBfNZSyxncDvHCMOMM4OE01R9j+G7IT0PSakvgEPDTD3/cUQrlEBUtEC1Qvxbwk8TwKWr91hRLbxIWmNE/aWgSg1BDJ/yyzgrVpD9K/MvVpMXoRmaBfG7yjazpjbY50aaNdmhiw2YwCxRsLVnQDNb32NxogUZpgbiWCj8s0aaFt2kssXlaIJ+15JdxoqIFogWiBaIFogWiBaIFarVAvDTUaqKYIVogWiBaIFogWiBaQAvES0OcB9EC0QLRAtEC0QLRAnWyQLw01MlMMVO0QLRAtEC0QLRAtMCM/iuXcQSjBaIFmq8FRufzBa7ma6bY82iBWi0wutYcdckQF2RdrBTzRAvUboG4lmq3UcxRUAscVNDSZqzC/Au5/rXNqHqwQG1/p+Fn6py7HuqNRUYLNDcLjOZPsvvHq6KiBerbAotTgX9K2j+U1hzVn07vCf7F4KhogWiBaIFogWiBaIFqLOD31L6A5vw3dq6n//7PnahogWiBerTAsZTtP/CKihaIFphxLXAJTffC0FwvDV6a/IeJi0JUPVgg/vbEtBnV/6jWVOR/5FyxqXQm9iNaoJlaoAP99sfKzVWr0HH/4+7I5mqA+u53vDRMvYX9J0sXgG5T0At0YrWm0JHYh2iBZmyBP+j7mTC+mdrANz/3N9O+x243cgvsSvv8CHBgI29nXZvXj4wP1DVzzBctEC3Q6CzQmhb9AvPC+Y2udQ3ToPepZqWGqSrWEi2QnwWeJLuXhjfye6zR5p6flvm7ujX+Rk2jbX1sWLRAtID/FXd4YoYFmqE5OtHnbyHuYfU4+PHHE1NnXH8ksRx8Bf56U1P4EYVfHvLSsDRERQtEC8x4FvDTwqFJs7+f8Zo/zS32RxMPQnP9Eug0G7AuBcRLQ12s9N88axPlAr0LbgfDTUHxew1NYRRjH5qjBVrQaQ/NYc2x80mf7X/8PkMzngCNuettksaF7wGEcGNuc13atjuZhtQlY8wTLRAt0Kgs0IfWvN6oWtSwjZmD6n6FprIXN6z1Ym0NZoGm9j2ApbDc5w1mvVhRtEC0QKEs4G9yHV+owmbAcgbQ5vhF7hlw4Jpjkz+l08s0kY77BSJ/x7s5fomqiQxh7EYztUA5/W7O30e6kf77p6Oj6tkC8TsN027gpvQ9AL9A9CLEv9cw7fMilhAt0FAWWJ6K/oYPG6rCRlaP3+fYEPwSZFQ9WyBeGqbdwE3p0qA1mlp/pn2EYwnRAo3bAunfmmjcLa2f1vkmpxy+qZ/iY6nRAoW1QFP7HsDqmOeVwpoolhYtEC1QjxbwDxqtXI/lN/aiz6GBJzX2Rsb2RQsECzS17wHMTMd+h/gt5DDC0Y0WaLwWWIKmjQL3oeaqj+l4z+ba+Ybud/zxxLRbvKl9D+AvTPIuxD/FOu1zI5YQLVDfFvBHE/eC+1BzlJemWeGt5tj56dHneGkojNWb2vcAmlp/CjPKsZRogcZngeb+fQb/oJO/atlcL02Nb0bGFtXJAk3tewB96fVDdep5zBQtEC0wvSywMBX/BC2nVwMaQb1P04aNGkE7YhOiBfKyQFP7HoD/JW8MxE+i8poGMXO0QINaYD9qG9ygNTauytrRnLHg/hvVQBaIh0JhDN3Uvgfguxdpzn8spjAzI5YSLVB/FmjuP5rwEwY/aXD/jWogC8RLQ+EM3dS+B9DU+lO4kY4lZSzQt9tt/kGdf6nfMrf1CBEbLn5T28273rpqCM+Ibt9lby3dotvNnRph2+eiTSvAY42wbQ3VpE2oKP6DqoaydlJPvDQUzuBN7S8pxktD4eZGkytp8+637VRcXNQ13bG+XW/djR9o3RziHv58h9+Ki4uP69/9pvYhrtBu3663Lcbl5aJClxvKq5xcvMvkohZ7hXAjcv0C4BPwZyNqU0M2pRWVrQd1+iuQfZe5Y1nmyZ42cNPlb5mnb/dbz2Du3EXceVxsFzF+s+VuXahvt9uP199YxDrbt1+3O/rk057Nut66Qr9utx2azzP55I2XhnysVXNeD9lVoKnYNF4aah7vZpu6Va87ZimuLDq25awll2gEP1Fw8y0qLj4n2yiVJcUXVRS1PCM7vlBhyudHaMXLFKq8Gaic5v6jiTUYq8/gh9rGbFDRoJLKFhWXlEyayKcSlcUtJpQ8hjNLZVHl+cydP7jYPrPh4g+1vu+tbb8tKqpYun/329zHG4VYZ2UVxRWd82lMSUlJh4qi4nXzeSafvM35W7f52KkuedPfA/AvtM3o+oQOtIUF4bsZvTOx/YWzwIQ/KrbnLwm9dOdLAzLvclvP0jLzv0oqi4oHFBdV+t8Wp2imJYqfmPhx5XVbLXPHAnd+MOD7KQl4+nW/7amiiuKXKoordy2qrNy3ZXHFO5OKWl5KGctR/g/8Et3JQ9/f5h6f6df1tv6VxUUn4l2Aet4qrqzcv1Xrkp8nTqi8hs2/bd/ut90/7N1tfPc9RbyTfIRPQx6rrCzam2dnp9zLhr237alm4B1l36LiypOow/n9TosWk/e/++3tP6OdM00sqTiP+vvxO3zfFhcVf035Hk6094b5iipbX0rYv2HyG3lOGfreNneWlT3dcs6ff/DTjnX4E0s8VvTAsu99fASHVQX++tCsFFoGA6G5yrGu048m3u621GbFlRUj7/lox+/6dy9uX1HZsmjYe1vzTpwjmf+1w4V359azjvPHaq8yelcxaCfg3xj+JebGXJNKKjbkctxyUlHxs/e/O2CEGbyUvN21yzrFJcULMo/Ht6woeZi5Pq5fl3vmLmn117zM6VYlxZU9KyuL3x/23oA3Nu92+0YlxUXzFk1o9cDQj/uP9pOPln+XzFvZomR25vVSTKH37nl3mzf/VXkS4FOHXjR7icqKyu+Gvb/NlB9NbbX8HXNMmDB5Y+bfBLK2yvVsoeKayrviQtljWstpSu/OMwsKg2QOhGk1THy+6VigpLJ4Iw5MPxrPaNh72zwMhxUVV/iXRP+lO+8cMJl3d89PalG5/r8SCHCYL1xZUrlYZXFJWUXryucqilrcx+b6FGUtUFRRtBMT8Ere9fXkctG1srjyqoqSyl1MKymqfJzN8cGiTkXjuAzszsH+YvaFIVNXcdFCHPArt5q9pEdJSVEfDolj/H4CFxD+9HvltRUVlXtYHhv1g5Mnt3jIw59DwcNkyZazlXQualE5gDasGNpdWTnTzYQ/4pkOk4tLNuPycv5mXW/pPtfP39O3ym5j55l/6fHFE3qSv+ebXZfUrS9tQMEvg7850FxV50sDc3UX5shQDXXPuzt8zfgxNpkLQ1H/pW7uQPQ8xX+3/ML0Zd//5FnSVvCSazho0+Xvb8Nl8g3m23JM3A4tKiteCZ9IvN2ty/18vnxAZUVRKe/wdyXfMz43eaa/V5tcVPJgcVHF/7iMMOcqhvfrdvu9JG3EvNywsuWEl5g3xS0nlKxWWVz8WFFF5WmUMQ+Xlnv6dr19h1B3cP2RSmVRxcUVRRUL/z97ZwEnR5H24e6elbhBcEkgEDeCa4IeGoE47u4ceh/BOeSwuwPCAcEiG4hhQRNcA2Q3CoEkQJCQEJeV6f6e/+z0pjOs7+zubG+9v98zXV1d+nZP1dvV1dVc/9f06zrmOR2T0ZGf537NiMmhtmcf7HkaQak+MUZDcnUbJqNBmglbfZJ7tutpajR47XkK90O5q29b37ue16G48DSoI6dkD5pv56dtQYe87cTswfcp3MTZQ2ayGe+59omu5XGnaE+cMnNobNW/iTmDH+BYi+h8V3eHpYttjdSIyMSZQ74l4MICmzs621aHM2XKrKFfKvLEWUMfZtOoxYo/9qDB7kvH8IjiTPpm6CIMoaKGmTCHOXQWzN3oHXGjO9N5fB2xnX6WY//K3Wf3Fst//2djK22frTJWHemnrfSrQer7owm91aVR8uzy6JZOdH8M1L/cueuxmhtxsui4/6U7fqXFqAGXgDUzP+JtdrNk565ph3+Gk59/P6NLt9qW0zcfIxnjogkGSU56e6ffpFmDb/HyGgwi3B4yMtjSt1uZ6R2cE7hmr8NweINHDb9Mzhl8cY/secM4uMOAjs8XGie2F90qY+WxpHE/nfIQbJrYiJjSkBzfOWsn7ItLrEjm4ZOzh95JmhiO9oH9umb1Sst3zsME+gDD+WwMoksJPjIWqZp+jNGQXMWGrZMNW32Se7brb2qN6OA1DFpeyaNTjTWiiRFofGN3ywzXNufYChpCki4UHH9ioDTGYGhOR/6n768wHFsR5dgmv+JdthdZHzgS5eE2gxmxvALpxUL86RaQl2U1J/FVfhzyXiZ3RtRryabAtRzeWHAOKcSegVHxeWwo2XEOp1No4HnOE0vzmv+kyWh+GkneppOe3lqZnOR061Jy5R5lOLfX4+l0zlvkbizQ4+Mi0eRcHqu9z3X5MR3t/xUdwME1txSDUI+tigQjMIcjb7lp6Yv7dh37iWtFD25QkLZYjyGiXnR0/lz3fj0OczI2fKFI9lo3oi1pLSgcbcPPtlYzShd71BU3TjZGI2kyfhTw85EzzsuX889WW8uYbSODRPsSJ+LJQHasaO5EHqdMy5/naqSvGSMP3VzXYiTOiuWrsBixH8e21fRjjIbkKjY4DyC5KddOarp4GVaLre1eOyUwuaacBuiwf+OxwpblLRgd6Za2a/1WWvhoWsEPHN9OQ61+OBrZPWzb+44GUQ1t0XD/sV1faEnnvrMXsWJDyn748m49290sPT0PJu4utu0uoIOZ63ixBjqWHHnrrtZqujLvJzYF0Uj0Rd1Rxu4qbe83GT0ndB13sBX12vCM+SI6oF0JN9l2nFMUrxrkUNKcC6XqsxryTaUky200jJxxbgEFz2vQyG3gV0CPp5jX8DHn+mnO1xVBQ1VhuO6Y6GsHjU35eulu5Nx019kGA/chRhf650fcB0/onNXOsZ0PeDQwOxpxr+LxVC8/n8KtrfyLhOtJIxl/FQxO37P1H380w53/R+vWG30/x43KvZRHZmf4uK6zj5fX8EXbsdbxfygyMDzPzfTjVcfWGA3J1SrtqSUr74DkJltrqeWS80zQxC8jRgMxDTiW/Ynjej3Lq47YxEbu6EoL//KMYcu42xrJzPapPPc9h7upxwm/a1pG5PmMqDOWP9YOPMN9klfQzk63Iq8T9mnNdo9ENQLhdu/fZcylpaUfPLbezh/P/lbk8bReE83Pi77O/vOTZw39iXkOd9Gw38Az5av6dhl7O3ec/RR31KIzaLTtOx3XeZE7zfN0jI7jZuyIny0vupIO6JG+XcdcQZrnEvwwyveG4rH/wgmdx/WRO0nSn3QmJimtupiMjMrOML18hY+NSi307AZtFP7Ibs825vy+xlD/k5bjTdY6HOL0Nk8Xddqcu52xHDYzSPt3G30ocxWyV0U25k7MHjIWw+Ilro2mjhN7s+HP7jlzn3x55rDZjayMU5VPNGNt4QiCdsojttXnxB4v7KagzKu5mM1r06f3KTI4ovmNP8WvmV1gd4k9NmOHvL+wM3KZR+FN4f8xXMav5uUwmnEmh6tNjNGQfNWGbUg/bPVJ/hmvdyna43lOfGxitd10Tw3tjUH/2BsHzFVI62B9GPSXmzSui0SdRb4/d+qX8ybFrTwe2I5O+JvcDQV7jZ8xaJWGgPM2FOzDXIQZtmtvz/YO3dUr3oTZgz+ikb+CVy/z/XT8LXdf/2Dwd0Hi/pvZp65z8xruSx6f0/TuQOf/T+44z1M4zUVIs9w+HGvIHdwSy40cTpgsHWPG/e2MfFzMXeg23Hgud71oT02smzJrWDZvgRzlxB7BeFszaXM4z61fUxwMje8jduzRRuFu1X7VXp8A9dlo0HX3NpT78RjXwVTbdQ8kjtXQyzxEW87f6VbUnuazunGj2AiB5jnIaGidsWIzI3di9rB3ifVWIy9jIYbgQq7RoZGIe1Puhmbyn8tkyAX46+2LduxPT3fTd4643gY689/Yj4nrWcu4flf7+2x/jEQLCrRPuJ+ZjDuOt3p+wn24m+ZdGAvnWX9w0tdOmd93DX+YEzF47lf+dtT+kOv275NzBuVMyR46hRQm8SbRty3+/H0B8VYwoPF7LL75qRMa0CgDjVFoRI2U7sSMGA0UaYCG62291VDkUYKDRWauI6zunOqlUPeHE2fiV0ERalu+qUL8MEQdTyVOq0hFTuiW1Z7z8EV54miUi1Gke0oPS9edROE/0pfyvZXEJKs1KTPSkHz1fkmSYZoHIIt7XzDXSvKvlTqbInfhF3K3dVZpFdAiUDSvPZjp/Whp4cJ8jLpfkbg+RRXqW98fTWSgO0Z+yrcKpK9nvZ3DhMSveWXxMN+vuO3AgVkRwp2Ske7cW9zxTX6bJutu8jMuo4GqaUBDscl8jlm10lQ9tiZedat6MiYFowGjgSpoQEPP9fl/eCT1V9taYdFjB+7oB5YW8fhuWW2TPP+ktOyKjul1St68UN2M1GMN/JO63xSi+v+PulwQovqYqhgN1DUNyFiQ0VCf5REqf219VkAq1N0MOVfPWQjb5MGw1ad6zrpJ1Wig+jSgRxOTqi/5OpFyuV+1rBO1MYU0GghoQK8FrYCwGGW7U5eFgfoZp9FAlTTAa4vPa4JalRJJsch6Jl6Nw8yaAKmJkPVV6uxIy8CeWa37ds3qGpYTF5ZOLdXOxzIKpFdtypxdnmoFL6E8WoK3EWxfwnHjbTRQIQ0w/Xwfy402rVCkFA/MqoCHeU7so1rJLukuJLgVfJLshOtQenV2lKGgwO0d+/5EHVJ2aUVNK+2gOVYlDfhD+uVaH71KOdVM5I/JRnc6WTWTncmlpjTQt9uYfWzX+RtrEOTywvhkviypia+WXhXMc6JDbc9pxgqQf3i5DZ/T++L9O4/tbkWsKC+799JHeqyI/Xp6gbWgIOKeRfyIlZ/xhNby79c5q4fS8Rx3H61twHYq6+Z/Jr+gaJ3+tNz1+kDV1vrCn/9ly2AYTWRr0CDtYMt21rC+Q2+Wb16c3iHynL9Eb7/uWXvzJcMj+FZFGmsszGRNhdhQ/gndxpzgRCM/8l5+X5b5mbSi9dazmy/77URWcuxAefJZqGoKi/XMOqH95KaR9A2H8PGsX3kf/jiMmp+6z5o76ptuHQezht/urPj35sScQbFOW2+F5K11T1Wd0NkcvivwYu/e0yPWn78fycqXW+kVOr5PMFkLBq1s2vBUFhLa1g+nBQJ010n6GR5LEaPbP9I72mPy53uDlA+Hl+db0RdezRmukUpf9GiCd/FLWE3QDxXurYyGG8JdxbpRO2M0VN95ktGgV3zC8rqZbwQZo6H6rpkaT1nfSKDzG80rlLfSabekAB8xm7tHWjRtXb6TN4PO9VGOLaLjG25nbtD1fCIP3frymd/zeT3tCRaY2Wi57jR6XxmVEzARelvpeaNxH6X+ES7FEPgvneV8Vqp7ieH7Mzf7pK8+RZ237j2+YPklhstHLNx0EZ+53p+P71xNGkWS3ihjW5bH5SuT3seW4z5Jmc8smOvKiD2nb7dxx1OGh1if95/kp2eC/2blyGaTs4c8G/Hs2zAOePPTfjdq281bLP+NSb12K+rEin7OrqT3KUtXt3FyN+j7Fs9ijHyEHqiHdSML9rC+hDuV/T+J8xbfK+iwZfq635eujb5DXWbj/x5fumT1ynEHrbS2vrKosDi0Mt/K5b+/gwEzF0Nqul5PxVg4ZHKOdbFjuccS93LKNJlvB0Ty5znXo7f9+GbA85Tn4HQv8iZJFH1dE7eMhtugvsrWVHw3+KCmFcC19ZDjWSzS5J1I3ltwzj5ovmb9YK0QqlUiWSKda9tqCFEuvUsxeF/hXNp8wvou4pzJ+V/PuZ6h1Zsk+p5EvhN9kifX+xOOFcutFybPGszkTtvr323cvRjEg3DrCcDcaEZ0WGyl1FjM1Pkxjyeq71z4nWz15VCzKYetPjWrvRTNzeGOm6KtdLwCfSXvQdbVP9x18te4abmNWCr3Kq2CyJcnn6HNu4eGr+hxG3fKn07OGXIz32CgcbR+5O78WeJjYNj/oNGLjTDEqmxbb03KGfoPwj1Pg3g9RsEVQVXk2yzTjDXACMQFCpO7Ib8/+Vyo70sEw8XdEX1FkLAvsqLjYDr5ofpWhZaSJu2TKc/jgvJM5J68qKw0zg8yGnDllJzB72MEfWSlZZ6qOqW79u2kuzxSkKbhf0nT3PUFrOY45Gnq/jT7f0zKHnoDy0vfh3uO66Z1+j232XHUrwGdwzkq73o7Vx36ma1X/dGSPN/k8cRSjTK0XP67Vi5sHPvyIOGimY0HYDydFl8hk0MaDRlyXmHaXkfqvDg93X4to33kYvRxjQLEZRu2HUErD9ZXkS5lSOXXtAIYvWpN578P35ToEc1ovAPu3VY2a3Rsvx4TW2AwvMj3Si7kPO7Mms5DuB6f16e2WSDqJK65o1nRtF2L1Rs6cG6b++VmKer7uT5X9ciZu3N6E6cd19le/btknaFPtnMND+MLqe34z+1I/Bwn15aRnnJiRhqq75R8S9L+PIAl1ZdNjaU8g5w0ca0JrK2xXE1G1aqB9Kb2S/lr3cOjViSHUYAlUc8a03pN3l2jFp2+krusJqxU9xoF4C7PU4MdvMlY5BcMA2KD6zk/a9/Oi+Z6aU6k6Ji3aQVDRgLmOjS6/rH4tiPr/Hcin4UB/4xMO9KW/eAQvQ4viC2ni0NbyvuLOvxcJ39Ouu1cw+TK+ynLVtzdRWiMX1QEie1Eil5VZGTgKy+a+28eIXTNt1zVpxWPFFjUJyYrXl9w8uqYi7tL6hyrU+Eha6PNhyeIrw5+96LyFkbMjEbdXePh/I06+nZF4fLWyb+B66bzgSNc3qbXJyOWe5trO4/n57nLrXmM2DibLYbVl9CvQ413mOSZKqJHE0Xns8YL5VkT3sw5NXYCGQXLxnBo6RTkHoiR8G18eWlr0sxBn/ftOu5DNy1yFKd3X66TF/xriWtNhuzpheW2MZKtJ2Z263iKtQ5f2/qJZcePXb/eezGzoZ22NLc5o3FjJvNhrHuTuChYUlUWbASSmrBJLKYBfx5AGNSRRyW+hn3CUBlTh7gGVlsN09Mi17RYs2ELfC6mweu/ommDy+iQT6Yz/bvjWjfTgHV2Lfv8oM5o6LABNoljs7J+sWK38L0jhV/GXObva8sXJ1fTcL7J3VpbH6ZL7L5l+qqcYLiY27aK0lJM4m0RcfOXMZz/BI9LdkqzosNJY1caZR49cDQuLj26nLHvCljuNO74386zor0J2xVvHj0UyWZ1wpj5S514LKLyTvPLqm3UdnbnGxpfFaWCw+UzyGzeSwyX4UW+jIWzNSekUNDB2nV23jHpac62ZPg0Iw1PBJbo1kjGRD9sPdw2oM59QMZrrQijRxiQcWFULOZyYjdPa3zv2Nbz+D6ElYlRwSfW3ZiRUejvrtwUzmuKQdtcc4Fi84Fc+3su5CkyMBj56o4V8R7X3VBGJH7Qx9k2xUsdlzEaqvdchG1IP2z1qd6zXwdSp3E6Kb8gOn5d48Zp6Z4znQ5xMZP+cnnMwEeZ+KBOJ+erjQ0apjF0cDnVyaholWhAh+krgppo6HmRa9gfF0zDcyMv09AeEfu8NAd4rnsaox4frNior18nCB8S4i7sFPkyBHwFneuCl3KG/4ABsTUN8Zdy61PFNLw8F/5rWRume43xbxR1I+9qoiFfsjyZ/Z2ilpfOtnwS8V6hvIf4SxL36zZuaMRzP2m8bh1tqZeHmbHlcd2ztnejDs+2rYO4+zxcCfftMmZwxPM+tZoxUTRBPC/9Pw29jL+P/3rQH5GC/KkcXo/uZaTLSNIS7vKrr3IoFdfNyopUUgAja7O5DnpoEq/KpTksXIN78+XMOZizM3kMV3Rz5dr2HoGyz+Y6mcGjrfjn1d3v+Eql17dzVifmOuiR1/0YmofjdQkX1LBAvJRxGqOhek9F2DrZsNWnes9+HUidbyM8yU37nPyIOwsDYj7P5X/KXd/sUc/JfIKOa0P+PHdGJG/ta0zYGkt15unTwtwJLcb9vV89DIEvLS8Su5sqiNgbY/MG4ge54Z/NFLEsJ2NDNh3qpxkdIg/rEHfUn1lOZM2U2YMW8Fx4EI8t7mPodxHPdU9nhOMYhmbVaSbKH0wWPCIWjm8QRL3IQAWw7cjf2ZzG44lP+FzwfZbnjqDs+qw7+XhfuhFvldwT5p7yK+lfF7Hdt/nM9sc08l2Yf3aX7vz4QucG7vg0MhgTRi5+xmqZX7Rv2197dtqf+iwxdRqAzu5QOWjcL+BxwzGaGJe7MarRhl8jrvviy7MH6Y2N/oxq3K5wGGIXUdJjxn8yaAOdxI+8yVH0yMQuKGCyqLU/jzIWu2npb1OOGyfOHPItaelZ/nTYdNfKTj0TPZqQAZZSoq9L8rDudSdv3Tt8Ev3aFsuXvsF5m9Uje/4022nwX/5H+3I+H49/Qv3CosJ7+gqsfQ/X321cr3cxcvWQHXWyvYLMn/A/MxaHT6xzUV/GcFVKTjr/qzVfVDvjSIIGdGe2HLaFMMwD0BC2OotWsPlQLh5GjAaCGuBO/mbmgTdh0uE1Qf/KuLUQFHf07zDJcIfKxK+jcfQc/2V4po6Wv6rFVv/0IxwGMqJqXDQClh4p+PWlb4Z/p8wT93mMdyTGQydGFub2nDn3rRHWiFi7OLBXVvP8vKgeLWG52h9wvC2fS39buxp549XgY3gDJz/NtV/25y5oUq+T5xzPnXxLPhH/UXGvJyu+kfBr4H2qqIs+LMLwmtUjLJUx9ag+Dcho0GtkychBRgMTyn5ORlp1JA29xqfRGxno9VU0rD+vvlY+VettHk9U/5kJ25B+2OpT/VdAfc0hzX3Gc73HklH9zMa643RijyOSkV4dSONIyqhJk8GJmnWg2Ektoh5NaKTFiNFAvdLAcdT2jRDV+DTqMjpE9TFVMRpIRQ2MolDMg6jXIqPp4HqtAVP5eqkBzQPQMGNYRnWYnW4trpdn0lTaaKBmNJBGNn/A9jWTXUrmorovg7+8bZKSpa1HhQpLR5bKp0wTIZdAt1QuZAXKplnfmbBjBeKYoEYDRgPl18AhBNWEY7Ub9VX05ogWtYrWVwWkar2N0VAzZyZs8wA+Rm0H1IzqTC5GA/VOA/V9QSed8BPAzGeod5e+qbCvgbDNA7iKisXet/craLZGA0YDSdGAXjPknX2+eFl/RQsmaW2N5vVXBabm9V0DYZsHoFXqZtT3k2rqbzRQDRrQSoKzqiHdupSkRhliaxrUpULXl7KaxxM1c6bDNg/gK9SmO6GmNaM+k4vRQL3RgB5NTKo3tS2+ouZVy+L1YnzrmQYmUN8hIarze9Tl8BDVx1TFaCAVNKClq3ulQkFqqQx6PPML7FJL+Ztsy9CAGWkoQ0FJPKzJkPsnMb3aTipskztrW58mf6OBTqhAbybV50d/e1H/P4EPkRlJRQ0Yo6HmzkrYOtmw1afmrgSTk9FA8RoYgLd5NGHemij+6jC+9U4DGdR4DYRlHkBL6qJFq8ziKyjBiNFAEjSgFRC1RkN9lG3ilf6GbZhGZOvjuTR1TqIGwjYPQLO8eyZRPyYpo4H6qoGdqfjvUF+N8H9R953iOjAj4CgiVaW+XqC1dT70xsF2IOMhDNKdSugZ7OdhqIypg9FALWrgdPLWyN3kWixDbWatieKqv+YzTKrNgpi8S9eAsehK10+yj4ZtHkDY6pPs823SMxoorwb0quXE8gYOYTgt6HQlbBHCupkqGQ1UWgNhmwewK5r4sdLaMBGNBowGpIHWsAI0aldfxaPi4rb6qgBTb6OBkjQQtnkAv1JRPYs0YjRgNFA5DZxFtLGVixqaWDIYZoIZ/U7xU2pOUM2foLAN6YetPjV/RZgc67sG6vujCZ3/9XAiuNoxkroaMBMha/7ctCLLQ+Glms+6WnLUd+81IfK1akndJGo0EH4NaKRuNOSFv6ol1nAdR/QpbCMprgEt2WmkZjWgeQDTICxD+ntTl8ehJxgxGihRA40aNVrGQTPRrUQNmQNGA+XSwPL169dvWa6QJlBoNBCmeQDpnJXV0Cw0Z8dUpFo0gNGg59ZGjAaMBqqggdr+H5k5DVU4eVWIGqZ5APnoQWvl63PZRowGjAaMBowGQqwBYzTUzskNk9EgDYatPrVzVZhcjQaMBowGUlwDxmionRMUtk62yvXp13Xs4/06Z/UIno5+XcYeiX/RBMu+XcZc3a/LuEHBMEE3YR+BFf26jvlH0L+87oG9spr36zbmsPKGT1a4/h0mbNG329gPVfYB3caaEZtkKdakYzRgNJB0DRijIekqLVeCXxNKEyLDMg/gE+qiT9qmQYXlhK7jDuZhd+NJswd940dWR2rb1lPsa9ntmORtbP6IZXvXDuyc1cT387dHdnu2Me4L3TSv86Scobf5/hXZ5ud6Z1OOkyoSJxlhvYz8I23Pc9I7OFtOyB7yaTLSNGkYDRgNGA1UhwYq1chXR0HqWZrBeQBvhqDuWjN+MejVS81vqJA4nneD5Xj3BiN5GXkjPct+ic70EN//9QXH5HI3/nZ+xD0bvwd9fxaSsxt7Wf/wLHrefOecAR2fG/nn1jv80WL5bydblr2HZVtzW6ze8PSoRWdsVJzju4/unOY5/TzX2t6zrZl5G5qNatjoz9au59F52636dRt3YtSOzotEIz0nzRr8vB/Hce3uk3OGju7fZewAwv0edbzjeP3o/Uk5Q17v223MPo7lnEQa6z3XeW7K7EELFO/ErqM7RL3IYOrXlDfQ35w0a8hm57vwuDfYtuxmefOilxLlAUZUdnQc+1TK19qxvLcmzBr6qtLCv78Tifzhuu6xEc/70PfXMSNGA0YDRgM1oQEz0lATWi4+jyoP6RefbK35Vqo+x/cavSWd+t7p7SPT/ZL37zLuTBaUXe149gTfz996njeZY8P8/cKtzQCB+xNuz3asRREnM7fl8qVjLNs+niPTidNzVbOGPObw7L5ds7pGXOcdy7WXe473Lp3+WZmNVt8RiWTkUo4VjG6swSBYluY6nRjVOMXPR/t07MO1j6Fxout4zxM2nbSbycggzmhslm/o5JdGHHf6Cd2y2h/d7vlmBZYzjbL9jEHzKek/2q/ruH5+mtrmuWnr8V9GBdY7lv3zgI4v7Gzb9heea6dZjv2Za9t3E+cmhbUdu7/nuTEjxrUcjawYMRowGjAaqFENGKOhRtW9WWaV6mQ3SyG1dipVn7R8a0+64Xnjxw+KqjrMY9jVs73LczcUXFZc9TK8SDb+e5zb6/H04PE0NzKGfXdi9uBnCiLudp7l9kmPOkMnzhoygdGBcz3P2m5A56z9Hdv1bNsdNnHW4McmZw990ba8ZzBCOo7/etAfGBJfuJY3f2LOoPeCaRfnpmN/cVL2kKsnzxo6DmPhZhK9fGLOkBcm5gz9j+fZTziue2l6ZsZWGCUZxP++Z/b8CRErekS6a2/2+OHl2YN+JP6HhFlE/PHRtMj5lGcCIxy3TMoePAYjiFXyvOuPbvdapsqBcUJ9hlzP8SztGzEaMBowGqhJDSTFaNAEshO7vrCLX3A9j9ad1sCBWRHf74SeY7br3+3Zrfz9VNiqTKe3ebpBRcqiuvJMXas6VlWqNA+gqplXQ/xKGQ2W57DYj72iqDy29Syd5PjMBpE9PNvtgX8TzXnwj4+fPWgtbve33KYlngMea7QnzfR8x32DxxnThOK4EWub5qs3LPC8yP74fQo/EO4Cwpf5P8AQoP8PiGt9H9jbnY7/hqK87NhjiyZ6REGsuxnxeOGbrh2WRa3IP/Ls6GbGTiCNmBMjZhdGNGQYxWTizCHf4rAzm6zattDHDeZb6GV+jQaMBowGakgDZTaW5SlHfr77SNRNj03qY3b79V563ueO5z6QP8+d07dzVielQYvN4+mMUXQImze+5cmgmsI4UfvVlU0aVGi2ekGeewGd0YgkFCk4DyAJydV6Egspgc5tm4qUhOH31UTaNLHRs9aQzKF0tDeTzumwNRdpbHhe6cbvuNNarc1dpf3ixZVu/2CuQR8fzIJ+663cqSuaNCQt7zDXdU7m2C6ubT3IhfmXa9LFk3Kk+enzKGPzSau2FRsZiR9fyRDA5X5eBY7Tz3LyrtGETeYlTMB/e9u1+jCC0JQRiof9NIvbYjCsIK0i41qPOAiXkbs2+mcsvOcE8y0uCeNnNGA0YDRQbRqostHA5Kz9aQybaeb7CZ2z2tHgXZ2e4exBQ3kM9sFIx/FuUelfmTloCZsF/buOq/HZ6dWmvaonXLm786rnW10pVLg+aU7BPO70d/MLxETBv/mdr+05l+P//aScwUf6xxs0Wq2wi/xJjb5/cNti9UbKYWf27zLmUo129esypjed9xcN3fRWTEjUMsaLNArAhMltmfNwPgaFHiEQxctl5KPNwJ5ZrSPML2D0oNvxnbN2ovPfhnBnx8IU88MgxDgeG9yqESjN0Uhz3cmWmz4k1y7YBePnA43CTZw9OBuTilECTyMlJQqPZsajj3NO6DK628D9shpmNki7m8Bvvr7g5NUlRjIHjAaMBowGakgDRXdSlc3PsZ0rXM/V82Rrj9lzfsju1q77+BknF90F0ggWGSZMGhtLA3s/QccH81PDzqjEfRgfx3LPF2VC2OTuOXNvGGGNcBm5OJmG/iqOtSAOE9XcGyZmD5uq9/U9x21HY90H/3bkMxX3FNz3EDaTYeEb9YwYo2awbTudLc/tQwPegob7s2h640tfnnH8+mAZ+nfN2o/n4CpDa8r4PTeaF9OBfa+y5c1zb+dWdDDhl1G+HwizNBi3Cm51ssdDqXefVUi/pqOqPgfAC+XN+KVvhn/HsP5qvUXwUs6weZvFcxkxiFhfBv24/++N/nWeN5Pcgsx8J2PD+/KUQcEI11FMJLyX6+oKz7GXOJbbf1LOsJ8wAu7CkH28f9exs1zP+hlj4E6shZNjI2DeuNcxHIbnF3i3Y7icx4TMkUxqfDePUQseGzxmW06HwkztuRgUv/oFaLl6/bWrmja6M992P43kOQWkNYa5FI/ICunbdewtrhWZwmTGxrr2XNf7y1wN2/Z+442L2UpvUvbQd1iz4caI5TyXv9ZtxnX3dp4VLZyAydwPJmD+4udrtkYDRgNGAzWtAdqkykvv3tPSWiz//U87P6PtxHkDlvspqQOgobyN8d0DbMfrHX8uy2HPpvFcmZ7mtItNPItHoNM4GudN6a7Th8Y/M5Kx4RU+q35d1Lb+ZELZh1HH2fvl7EELaYDPo8CX0aB3wmi4ivSucx3nwEhe2jIvPXc+jfpH6+zcYQ2s9MMcz/kPd6g7YjRczZDwDTTKB65q3frb2Kx67l4n5gy+rl+3sV9brncFk9/mYvzkEH+gJsHRaJ9KZ3Jtj5x5Xb/p0uEizJ7BGenOsRvzrVYRy52mTosyXOrXtwrbtsT9AHaoQhqpFHVPCvMk6NXLcgud8/kYgDvRYd5QViSugU84l6f4rzSWFd4cTx0NaM18PrRT3jZH19A+kA6z4D2oKdH/8TDQf5N5L0kR/Tc6x1PSTZUeN8kg3uzmJX48GZtDSWQxMLpVq6LzfSo8D2U9WjucMMxHir223ZJtsr56uVM83bfYJsoueKj/yYHYTUdigCrsO8R1QfORegH9WtWlgv+jqmeYkIIqVWlp9fsv2xM5GjQYlFiuk7bG8qIjuVo+5c5QIwtxoSu2rIW5BV5H36dw6+jOvUue495iZWzsmdbBOZTO+5Mp2YPmp3tO2zRr4zpGAg6hs96RuzeNOBSK7b2lMIX52/PJ64U3s09dlxmJfMIrblv6wRiFGDd59qA506f3KeCu7n6O9fOPaRuxIsey+cXzoraGsplg/yP7237dtVNnRif6YVg8Mn7GoFUyXKjBU4qTJFlIOvpTtUlSerWdzDcUoC00r0hB0jz7KUaJDihrgqlWiOR8fG4Mhopot06GvZlS61rS11P/DdNhLNSUyGAZBfslMcMhpMWoVmyezmVsH4AlcBRUhyiP/asj4QqmGSH8KJDxV5oczMFxsA3sDmdAskQd9tUlJDYd/z2hUQnHK+vdm4gPxSPvwvbcuLvOb6pkNHiRiBSdm6gFzV+YOGvYW83XrB9GR39EbK6DH8jzciN2dLMTNCln0AweXWDteVvwnvtzBfPcxf26ZvU6tusLLZl0+DITKKczk/4sJUGHr042JriKHoMwXOwyBK1vslsbnQIZJ0XCE5I//B2eba/iYHN/X1vXcluxaU6HdFoRljU5zSrIJ8cWsclpfgTPloGTTPmIxA5IZoK1mBZD87G7pwo1trwRkZfvOMMKIhtLfVzGqNJiN73R9bVYP5N19WtAbzNpxOld0CTQpvAm6PFg4qTlogmjHEsUXUuZAc/GAXeis7RjwbAqT0nSsKQDAf/JuPvEUUd2Oaij1F11UErLJxgu6C5PnM3avWBk3Ju1yYFj8i/pf6lzVZqUlKbiJJZ3V/x0zv8Bz8IgSJRNE6Y3P5KYln9UhkpJZVcYXR/ciFrnw1Qoj5RUBumiqG/C3QF8fauNPwGCUppeMwgoUlKqZDRYBRlLqVUr5h7E0mGlvON4VvyeX9N1jRtLMRE3zdrg+9Epbxm17d+L9nHEX6nbWu/TM+y/M5YBw1Luaele5GQ6bVuPI3gn/lTHcT/krARPTDCZkt2e18k/yPPtHhgBc/x9bZnzsIDNBh5HnEVeZ2yVsepcDIsfHTdN9WOintdN4SRYJh0LXUn7DZPRIKV8CBW+w4kZmtmnSt8likaVEueilBjYHKirGnApuIxP3W32BTX8Z4KGrbNBIqNiPagd0Y3CzSC5CHTDoNHN1XEUVp31WlD4fUDyIvwAz8AaWAHDoDjpj+dvsAoWweEgGQrK7wVQeW6FiojyVhn8DuUw3N+C2oTv4RiQfAW+W/u3wb/kQJTnEvgYPoDtIFGU/kL4HBbBAJBo+x58DYqbA7uBZAvQSqQzQWV5GGLtPFvp6yVQnX194iySQbh+BKU7usi30HETG7+8KvNO8De4B44GtR/HwfsgeRaegO9gEUwDv+M+EPdckL6kxxNBoj7iEVAZdPxMKE6y457z2Gq0QTfAW8X9tNF+KzgCpoN0NQt0LRwCEh2fAnNA+akeulZvB103/wOd1y9A0hreAD/8k7gzQLIMdF5VnqVwJ6ScOFUpUfyxxMLYMD4JrXPypvEP2po5Cg+zSt7QvIgrZT4Vf3PC6tdjYgv2W26TtlqKL5KI661i3sGjzDG4jPkEZzOCcDCvvL3tRexvmUneHkNkoOYZsEjwCCI18Y2UogTKcJDesTwLv6t/1zEXMcHxXst2NzsZW2Ws1J9j7Tdd2z/fv9u40/7IazGOq25f7oD/xEogvH0VZbiyPxPUeHRRUsNSRilKPKwLPiwjDarkx6A/sxGjgcpoII9I18K28ByoIR0Lu8N62BJOgpGwJ3wG6ogywBc12n3gJ7gD1ECrI1LcK8GXtjiUnzqtX+Ap2A6CsgM7o0GPS/YHNejjQSMgvvTCcSnIv6Iyiwiqm9rGcXAydIXj4RlQpzQK5C+haYq5n2YrPciw6gBdQO3tYxCUrdh5FgaB9KKOTHF3BonKPji+/R/b/4DkIZgPu4HKp/RPB0kjULvVHGZAULZg53GQvpXfdPBFdRoGHUHp6bzKIJgKOuevwIGQKLviIZ3ommgJSkdleAnOBx2TIfco6PwNBLWp7UB5lSTd4wdUxy9LChT335ftBdAGpBuVVyLj4DdQGaUnGWg6lzfBRDgbgqK4MoB07e0CGum4Dnz5E4f894C/g+qbUlIlo0E1YbmZ5iUAAEAASURBVB4Bf5ToMXJrPkG+FeUtBG8hB/Z0POshJiPqpBaKm3c0MaaMnHFevu+l7cTZQ2ZixB7LfIHmzFLfwXacs6ZkD50yeebgN+iwz2VOQmeONSPto3kr4uzZvTo1jUaiUx0voj9Dodj2nZYTJR1uU1a10FLA58SPsPGe45+2EKNjSx5iHKcZ6jqGQfJ/jGbMU3miGY17s/8eb1C05UnHq7kbmunkW3qVNI3VBalTY+Y2MFfDOYLyvKBjSRI1RrqA9AcMg3xCJfaEtFSszPHdstr27Tru8IqVzbNlTAYXK6tY/NJDa77OiT1eUMNlpFAD/2ajBliNqf6r+8BkOASWgTrvN6E/6E5V11oT8EV3mZ/B13GP29jqxkANcsu4nzYuXAZK6x7IBOURFF0rDeBbUJnUkapTUKfky79wKM8c36MC2/R42IPY5oI6udNgL1gD6kTV3tB2xgyVg9kuB+V1HCyCAaA4qs+RQHNXJEpX7aJ/pyudfAyHgeRdUN0kz0AfkD6V9mpQukPgF/gb+DIOh8qnDk5hhM6H9DIXskEiQ8Rv7/3yKpzCSw4FXwcxj2J+puC3EZTOLFBbuTeonG1Aaamey6A3SAdZsA4KQGWoqswmgTnxRL5iqzJIjgCl78EG6AC69koSle0RUHjV6TGI9Z9sJSq35AdYBU21k0oipVdJeKvgofyo+yYN6n1aCvjVnOErSPCBTYnqeouL551FZ36Jvxvcal4D+2IzmZw9+GU8hC9j4w4ptEh4rPC6v6MPG+F+3t8nzwJenxzp7/vbeNqx3fiw9+P+seA2/iqgGp7qEF3UX8J+MLU6MqjhNHVeFkIPUL1SSiKWty//15Mp1NvlLdjAgeOd/HneKF7Q1B9aDUNShVeWz/cKHJXnu6QmXDcT60Kxr4Dn4J9x1NC+AceDOjg1yupAXgR1ULuCGmFf1FlI9N8S67WDyB3sUNXJ5ukAog5QIsMhKE3iO+3ZNo27R7FdCS3j+7/Ft5XZ9CSS2rfG4JfTT+cWHOokl4OuDxkH6hyfBklinGX4nQeODsalIdvEdLXvd9Rr/YBspQv1CUJp+3rEab0FMhx88fUlvZwe9/yJ7SuwMb6vjdJUeyxJTFNtxdkQPCcKlyjB9HTOJDovwfLJT9eL+hB1wsE4flnxrpAE9RhMzy+DEks8B2Vl0IgAwfMRPBeK6+tKbl3TZelG4WpUdHFUSfTqJI8Onsib755BQv8rKbH+XUZjkdlfsua//uQ1Jjz2wFr29KdLZfmQwu0PU1O5kBUo20eEPQC+LCuOJrumR5zm0UiTpcH5CiOYJzOjW6edI07Um/TN0EWJ6fAq7Y5OQeZ6PSLjrYuM/PTodolpKE6/HmPapOdHVsceNSUmEt/XR6IKIvbGKbOG/R4MUvg4baPuKtUYlirHdc/a3okWFCSmoYWiNkatJr2y5yymTkWNjb6dsTTabHuV2crdvO3T0uYrWzTYRqtABhd18uPYbv76iWXM/yi1sKl9UI3z6dALTgF1VEeB5GegHYkN6Z7IdkIcNpUStX8nwyjoC5KZsF3MVfjzTdz9BdubYDjsCTKM24KkoHBjbc/2cND1vyDuV9rmbA62gpdhR9gq7taIiMr2P3gfJDIULoPu8HeQZIP09Ix2kD3gTPD35acwGgnxO+xM3PvE/bZmq7pEIAoHwizQOVC8P8BP6yLcKl+iqOzCl91wdAM/v864fcNLafYBP82uuC8O7OMst+QQcgt4E34FdfCPw+fwFaj9eRAkqm95RPVuBUuhI+gclCXSl9LXVjIZdK5K6vClg4PhBZDI7ceNeaT6T3mUUmYdJmcP+W9ZgfQ2BWFEjQojECNrNMPKZfYx0fyGoHIppFYsNZpqhB8qrViaw8Jf62Kaq+/Souu6sfbGlfoQ04DO4w74hq9IRjx3kRW1t2COTAZrgRwgA0ELQZHmO7AFy5V345PU9+R7bj/CuZHo+u35nsg+U74e+gvh1hPmY5rz9XxKuxtGxr/5uNR9wfJotUcWb3qFdTr+ZEGxLXilc/5WmauG6nGV5r9Y0dzrMXS/z58XpQGwg1GL3OSziom173iuuzWfsNiVRx8vTs4ZfLHWMGm+/PdR+QUuH8myfuHbE9ud4Iw5ccrMoV8zYbjj0jzrFeba/BqJrmsZS9orHPno223c8Ss97z/oZG5mo7T2pH8f1/C/9fhiaW7kVS08xuP7ndHdXD5odRLlUuMUJlFnewuMADWwvnyG40nYBjbAY3AzNAPJjoWbCv1GCa28HoamoIb8awgaDR+yPwZuhJNhB5gK6lh88c9BFzxGwemgeiSK/hPqjCTbgvLkHMZGOeawVf4fwHhQZ6J6LgLJG/AETIc/QfIf0H/tRZgLp8FtEBTpUJ262pgJcDxMA8UbAK1BaX8Bii+jQ6L2SPWWkaJ+QuXsDWXJdwR4Ht6H10CPNPJA8jicApNA5ToV7oXKyGIi/RumwzjYGzJA+f8Ep8No+ANUZ/mXJa8S4FmQISL9L4eyZAQBJkInaAm6DhX/UDgargPp1pfrcWTBnqDzfyQobJ0Rp86UNNwF/YTq6SLSnzMMogbpgDIr4lmXcIMwhE7xGCa+nshrtW0Vh1UPh9AVXoW/vh3RDa/VXlr+YfH0GmtlUfwPptO9gUdPf8egOJr9fWm5Z9lR+5h4uAY2Fj+PpU7ga5f7MhfmOo1OxI/FNhHb/Tcv54yZlDO0N2/OdGdUKn1pbvPztbw081vusAuiB7Cq6CEYBWqcS5JMbipeIv8DnILoPszJuUDLP7Po2ZkYA22YG9MRo/pAwtxqu3w+G6GcD3DsEflT9oPZbyX/E9pPbspE3Wdt1zqe9I6y8zJ64X2TvvwZLUhTo/0WBsnRK7fYuhtPr38b2HP8looXQrmVOunZ8OlxdO51Pa0BNf5qoK+BK6AH9IGloM5I7hyQ3AaHx1yFPyeyuTqw7+LuCReArhsZBZJPQem8rR1kGGj/ZlBHoE6I0xYzXuX/GUjUOWhfnXCiPIrHcLgljvJqA++DLxfiOBsWwp1wHCgfiQwcdTBXaScuMh5UfxkbC0B1fQIkN4I6L8lZwH8tZoBcyVYdty8quzqyeXAAyCCSvAPSzQz4EKTz+SBRJ7cu5ir+53K8pWeVSYaSypUHq6AXPA3fg3Tp/7dkYOi8S9QeXhpzFephStytjfTi71+LW3pUWg+C8imAtSAjQteDyr4f6HpJFJVJ5ysaPyAj5g6YA/3gWFgNX8LF4Etw/2M8paevQeXaF9bDK6CyfQ5fwfkgmQY9QHp9CzqD9CSRrn6LuQp/EvcDh2rPmVZ7WZucAxrQn0kNhS4mXZB1XRZRATV2bUH1KkleY3nvqdxNv8JE01fzNjR/UAHpMC+hg99f347Q2zOkRDounXOhZKRH3pWLOD/b/MHjb/EwPun9yjocDePBoq0zVmXJzaOJ3+h4v3Qc6wAVqkhs6zAMlSWMcNz8TczTS2dzSDSSxvLSXs6EucMXy5vFxiawHLUamGIl6rlv64DCU5cCmpmGrCJ6MGUbH59fY22VsfqFpXnNn5BBwkeyDnbyo+cpjsrOyMObKlckc91eTNZ1GE3oR5n6eYXzx9YymHCQY1vTWPb6LdJvby1b+jp1vzu4qqrSCpl8S31EcbIIT+HLdN/BdknArcY/KGrgE0UdrzrdoCxnZ3rQo5h9HV4aR26J0pouRzGijk2UJeowRXEyuxjPXPwmFuM/K8FPxknQQAkelrEjEkW6fD7Rk/33ivFL9JoW8Pgl4Nb/aHJg33eqs/Q7TOlfSGTMBCVxv6SybyRS7P8fj/x7MJG4W0bj9IC/DI5g2WRQSVbEie0Us6/6FaenqX4EtjISfClJrx/5AeLbxP2Ew7Wz69ROtsnLlVchh9MhXJK8FGstJV0gsvTDImXWB+PgUt6wOYZOcTEjBjdnNlz9gSrP0Puz3PU/wj86w7bcSdyVzwwqZe2aPDWUeNO/Wna+f4yONGgTuCNnnKtGoEgwQOjHN5N0LI9FfLCqED48RYf9GNYH36squvuwNMGXWJulFUwlUtAgeMwriKZRfEaNvE1prNilpRooL+JkqoyOle75dzcWoyyx+rh2JJ2D64vKo3LZ9m1M3vx0QvaQT5kwyTdWrLGkfQD+cwd0HrtXsBzGXSENqFN7tkIxwhdYRrHudo0YDZRbA3XeaOCrxjuxYFO7ctc4dQOW2cmmbtGLLVmp9dEzf+6ap3mRvMW8AjuC75EcSiq9NLRPZztE35fQHIS0pmkfYh60pxOPFJtLyZ7p/buOP1iHNdkSw2RPJ7/g44TgX2CsMKlw8DPC9bxdMGLaMfTPHYbdXV+sVPgB3cbuy6ZRQtxSd3m88jkBjsFOiBkqjFQcyf5v42cP/J1O/2PPTde+pcmN5HWQ3F40+g2BmztOwccqz8b1TcdiCJ0UddMaaJ0QFiHrxyON//HdlAGaR+FGYo+0FNVIxTWgYfGzKx4tVDF093t/qGpkKlPtGkir9hxKyUArSDIJbabt2CfSODbyrMhLWvVPUY7v9XKjtLz1w7l33I7nxDm8dcEQHE0xwhcMO/EhLBpOrfDmcXcWa5d1yOrfPetAL+odxh3qijzbfS7+CqjFwkxd6IyOUxjXcl6dnDPIf+4pr1QQdbL3pEJBklQG1afERlnfAeFDVR97VsYHGA8fM2GwG+fnkfGfDNrA/iTbcUexINc7eevcgzi7P3JnvX1Fy8WaG49wjX3NRcMIjv3ghLlDF7Po2P5+OhgJl/Ghsil8rl3GRQZzJTrmR+w+r8wcuqRv1zF388XKz/t1HfM+jwW6kIaGO8st6+y8xxtaGQP4QBtGz9hFRDwM4/a02DUcHcuXN63XSfvwpbnWrhgRMeNdb14wynJ9NJr2Hjp4g8ep3Yj3XY/Zc7Jndu7gMQLB44kxzLFwuOa9DraVd065C2QCGg0YDRgNJEEDtTrSwD3Y/2EwTPFcj/WT7BaO536mjxYd3e61zEjeug9p9PvQxf9MuKtoRP+n+jJbfh8Mhml0MJpZvCuN+ZW+HmhwL2AW++Oe4y1hmHn7dCvyycBeWc01kYzwU7npW8kINp/Zi06T4eHHS5HtIspBday2KVKeqhZDjxR2hhYlJYQheGPUcQdwTp5heyqTFi9XWBYEG0zHeJljOVM3WHlHuXkN/5YWcUbqGJMNd3l9wfA1cmc0tt9Oc53Bckvs/My/u/kNR8V2tG/nHU7n+nQkEmWi5ODb5b/eyp2C/xlyT5k19Mv1du5uPCd4hMmH9/DmRGd/9VKWNP8n8Y7ijIzi66u9XddpP/6TgX8xHChP++7zsvXMMyb+vhY6Yx2QgxyX7wt43hPpGU772GJlhNJiZukZdkfSHslkzKFMhtyfRcbGKQGWS3/IsfP3lk54HHEG+0NG8KpmLI7Lp7mJQ1kfbLFmQ/cQv3YZ06X5MRowGkg9DXATV3uCIcAQrvdfZq+PUim4s5zrOM45bjS6PS3+5ZNzhuwnf80qdzI2LOFOa09GHm7ljvAzGvUHdIw7rye5BVub0d6+kiHgpTSohxauMBk7NpqxiU/d2Lcl+Lql6xyvLyTqY1jprrVEE+SURgqJOo6X4fkUKlNVivI2kTX8+XpVEqlMXK4tF2Oj+ZT5fWMGRmXSMHGSq4Ha/qRvGbV5luP6/+kZv0Y7O8JfjET8qlt4XBWbAFob+euR2SlxqrueL5BBX3gM1OYth8TJm3glRb4jlT2hO1wB/SEZciCJ6O2KbPgX6EbpGahWqe3/Ua2ONEizfKr6R1/DWDBreK7LLDJrdzr7r3z/eMM/3456u+HX3vG8oouLOLFw0fl523KslRexJtNhLBSkfrRm32+dsfItnlV/yAevshl1+JZlovuvimxMxc7kI+rAUHpopPbq49m3ZKbl5oZGk6Yi1a2BrcmgEeTDGaBZ/rUhNGlWG6iNtln1lx6qWxqQwVDQSOTVcBdsA9Ulykf6zAG9hpss0ejlDvHEtmTbNFkJp3I6abVdOM9lbCBRWMWREYXExwet+O7DSuY3rGDuQ8tNUezYiYqkZ6xx81w334v29Ocx6FEHq8ZHF+U3zGjuOdc2XrvukpVNGh6MUXJ7Qy9DE+uu35ROSrjUyZ6dEiVJTiFUn1rRMYtE3ZKcKphUakEDh5HnEFBDr1GqF0HSBi6ArWAtaDLjPNDcj52gHehuUq8sToJroTnoblavMutYG9C78bvBZ/A4eOCL8jwEPoQuoHR3gZ6wADRythEU7iLYA+bDXPgJim52cEu2h8ugNXwPD4PuToeBOjEh0b7uVHVXLDkU+oPuwJXn7yDRTcXpkAvjoQ08A4NgRXz7JtuJcA7sDZI3YGzMVfifZJQ3tm7En2yV/q/xYzbb82BfWAT3wnrYBvT4UEbFYngQVoJ0r4m8OheJ0gwPxWkLS+ABUH1uAsnFoLTU8Z4Kv8AcOCGODLinQWWV6HxOg7OAp4qx86q7/ERR/3ANbAtZgYOtcO8BX0Ff2AAngs71c3Ac9APl+wx8CpKGcBm0hx/hX9ALdobhoPPui/Sm+o6Je0TYXg/3QG0ZovGiJGejCz/lxI06r9KxH6X5Cyocjy3OYJPOe3BfMPchi/mQl+mRBUbBNl7hH8gaP2PQKsJMYx7DrfqwkN6Hz3fczwty3cMauZnH4n7z97T0jB6z5r3NKMZcmglddKkmajR0IbZItYJVsjz60+nPlV7J+CZa/dNAG6qsBnsCjAZ1WkeBGm417up8dFwd2csg6QYKmwYvwd0wBRT+W1A4GxROHaduVB6Fk+EGCIrSGAG6ZhX+BWgGSl8dzY0guRMGgI5ngjqnnpAo6jyWwRPQHvzOVZ2NjBJftN/J32F7KTwLBfAOqFzq8FQ/dZxCx08DySBQnTQKq45ZndQxoDCvgPI9ECS3wLWg8Go3lb7qK+kNaoPUiR4GCisZBetA9VAn/yRIGsP2MdfmPw67b4GMLp0v//z5W7xi6z18w3Y1qO37A86EO2AcvAsvguotuRFGwARQHWUIqVMOis7zVGgCqsNFIN1JdoWzY65C42Ak7nmg8yOjRTrLgrfj273ZSqTzTqA6twSVbSFIdznwO/iyAod0rWtCIh0eBaEwGFQhX5ly17jw2tivzFPY6Gfs70/KHrSA5XRP4+w/xmMGWYuzmI/wN+Yg5I2wRoz8ukuHHZyMjXPyY6MO9tuMVfypNJg4NpzJc/9hbsOPViR9HfH/O3HWEC4wvlLYNWvfzIZp37Kcby5BP8zbUKALJNVEDcQXsB+8nmqFq0R51Bh8Dz3h80rEN1Hqnwa2oMpqcDPgPdgL1Eaoc1CH8iZI9F+5KuYq/JnN5r74/odsP4KJoHgyItSJSGTIal9yCUwCdVIliTrh2+MH1Zn+DdQhng/dYTGokzkEipPWeDaFn+FcaA5liUeA0+EXkA76wkEwGB6G0SDRzcXwmKvw51k2D8X31c7pv/cbSKffQVuQblT+80Adn9qbAXAwSOaCb0gp3jnyRFSPZqAyXQzqPCWfxIntBH7UhimO4qs+00F5HA9+2yY/idqJmSCj4Rq4DGRwSHaBC+FsULmvhRzQcblVDnX6vvTEsR0oDeWrdJdCcZKF5wPxA0pTefvXVxvcyldx94WtIR8+Bxlo0t1KyAaF8WU+jjlwHMjYOAWegdBIWm3WhPfN9WcokuB+MV+3jIXDaHCxpW9iR2wm8Y8FnbSZZ2zH9ibmxN6yuPKvx1LOR43dAeD/sVKugBUskF8f/dmMGA2UpYEZBBgBd8BYeAMuh0XQCe4DdRTqDGzwRZ2jL3k41AFJouCCOhzJt4Wb2O8ifneAYDqxA4GfYIcg40XpKH91oIvBF3XQxcnJeKouV4M6lNvhRUiUYBk2cFCdsy/qoFTOneF935NtYp4/Bo4V4B4FneEn2BZ8kX6Upi+qx46gTjB416xy+H3EGbjvBHWIP8Tdo9mWJEpvAXiBAKr/9oH94pxt8XwSVH5fPvQdbP3zoXOqesgoDIp0tAj8fJfjFsWJ9OJLGxyPQTDfT9lXej+DDAaJ8nw+5ir552kOnQy6dv8GF0FoxP8jhaZCIaiI38mGoCqxKoStPmE5L6laD3UqH0AX2B3USI+Ao0Adr2401CFdAsGO1u8k8C5VdMfoyw44foXS4hZ3TJ3rWmgDvqizS5Q0PGRgqOPQXfvD8BzIX51eOvjSynewbQQtAvuqr8r5M7QBXxLzVJq+TMTxEiju/rAQ/PZe+Qbzkx6WgKS4+tr4S28ngOpxB4yCxlCSyGhLNBCUT9AoKS6u4g0H1U0cCndDeUV6Cuabyb4MvOIkqK9lBDgd/Hz74FY9ZXCo7jZItJXx5Nfd99cxX7JwHAyqx5uwGkIj/kUUmgqFoCKybntBegjqoip8BAeEpC6mGtWvAXUsr8JeoPZJndgvEIEoyIhQI+53JBm4KyJHE3gAtAONWjwFFRWV47+guIfDjdAbEqUAj//AhaDONhfUaSr+D6BR0V3hNOgGvqjO98BOcAU0hPfgUZCxNAxUhxFQkkhfG0D6ORX0H1QHKlFHdz/sDBeBDBSlX5KoPPfC5bAlqB7qTLVVuZVGonyARwO4HlSP86A9vA6JonIeCFvB0/BP6ATdQdfC3lBe+ZyAa+A6UL4qd3muEeV7F3SGrvAKyNiaDUvg/0DpXQV9YB2sB4XZBoKi/F+G22EUhEqM0ZB6p1NW6ffQM/WKVqkS/UisfFDjaMRooCwNfEaAf4Aa8BdgIdwK6mxGwRh4Av4H6vS3gx9gGvgi93f+DttnIC++/xHb4+ApUEd5C0jegkWgDn0UqMNPTDe4fyPHJ8FwUOf5ZnzLZjPpz96hoHJrlORY8OA2UMf7NHSAy2ARKP8n4Et4HvYCGSb58BUMht5x/sNWeUtUZ7UbvijcKfA27ADDQB2dL3NwPAcHgNKXfvRfVT18+RnHG/GdgWz3BdVDdT4apKPGsD0kitJTvVU31eMgOARWgOqic+LLv3AcCPuAzseL8F94II50JFE6G2Kuwp/Effm6oLKp89fxhfBvkJ5kfL4Kkg9hXsxV+HM7G9VNYR+Ob0ey9UDXy06g9DqCDDbJQyCDZj+QkRRMbyz7G+EdMGI0UO0a0IWrO4ywiP6Mp4alMqYeldOAFqWpXMykxVIn+lKSUnuKdHQ3LGkEi0EdSHWKyn9RIAN1Wg8G9svrVGffvLyBTbhKaUBGyIhKxSwjUm3/j9LKKJ85XDsa0N3QiSBLOwyi+uiO5tkwVMbUwWgADeiangrZoLvPCaCh8eqUj0n8NdDoRcM4Grkwkjoa0MjLLHCgV+oUy5Qk7BrQUNivIapkT+qiP5KReqyB2r5DQvVq0Fsn8RTobr0zbJHENMtKKkKA9rBLWQFLOb4zx9SpGakeDej8aPSpWiQF/kfVUi+TaNU18CNJ7Fr1ZFIiBTV0K6FlSpTGFKJWNGAau1pRu8k0ZBqo7f+RsTZT94Lyh/RTt4TlL1mUoBq63b/8UUxIowGjAaMBo4FU04AxGlLtjGwqT5iMBtUqbPXZdKaMq7Y1sAcF0Ox+iSYLdoq56taPyt+jikXesorxg9H1uuTT0CTomYLuKyjTVrAtXJqC5QtdkYzRkLqnNGydbNjqk7pXTv0rWRiMhkGctu5VOHXnEPcfVYhfXFTpVY8WU1mupHAyGmTc7JvKBQ1L2YzRkLpnMpui6f3qsMwD+Iy6qBEqz0IrBDNSzzWwI/XvDeoQgtKOnd7QLehZgltvh+0Feiymtw1KEk1oPAT0hk9p4VpxvDdsB3Kro7JBZdVkSOXjd7Kaj9QbgoaAJmIqnOL3huL+25pEdzCUVo5dON4b/PURMnHvBs0gONrQk32lpXx90URQ7ausckuUl+q+J6g+Ehf0SHGVdhC/zKpXSeKnI537evDDSg+9QefPF9W/KbQB6d+vs8oRHHXxw+2O/0Gg+iaKXnm9Ju7ph9eE8t6g/aCoXe0NLWAbKC49vGPnqjdbrTXhSzoOjWromukNSsuI0UDKaOBNSnJsypSm6gWZQRLmbqDqeqyTKVRgAtf1VFCdgIbHF8EQkDwFuobkPx+eA8nZkBVzFS5OdBLuRpADWpdBx36CNpAo/fH4EZ6BN+Bn8DtTnEXSB9fvMAZmxTmTrYbxNcn3a1gIXWFkfF/lnAeKI1E9VKYv4Un4FQ4EyXjQjcLbcRaxlWGSKLfg8RUo7cVwPqgzlV7mgEYcZBS8B5+B0lU+Mgok2pf/d3A7dIIfQDp6BxRPupP4jyakX9XDr88dOpggbdlX/V+AqfANKJ1MeB+U9iiQDnV+JXeBjk0D5TsbFP9F+B7uBom2X4DCjgaVVwaBROe1C+wBiiO5ExRW9ZwEK6AzSM6CX0DXjvS1CPaDRBmKx2+g/ObCM+CA8loMOt9PwI9wGtSYVOB/VGNlMhmljgZupij6A4RFHqYiV4WlMqYeFdNAORs7dQZq5P2Oey/cari3BHUmGSDZFTbGXMUbDeqM1QmlxcOoYVdaiXIvHn7HrWPqnPrJkSAyFI6J+zVmq47jTGgAHhwJEt3VTgB1lpKdoSDmKjQaluBuEt+XcfNp3D2e7ZNxtzafwMDAvu9Ux7hvfKc723PibnXED8Xd17KdDHZ8fzDb+XG38nk27tbmXTg/sK9O+5rAvpzvgDpRyfagvBLlPDxUb1+UhureDR7zPdkeDzJ6JHfBdDkQdchL4RLtIDJyFsRchUbDNNwKI5Gx83jMVbLRoHr5IgPhJpARIwOvHUhUPl1D+2knIA1xK1yPuJ/OpYw9nQ8ZDfmg60/SEZaDf13Kr1qlnP+jaiuD/4eqtgxMwlXSwEfE/keVUkityB9TnEFwf2oVy5QmhTSwJ2XRXdwf8TJ9wXZY3H0RW3VOu4PuLNOhJFEaebAI3gB1aEorUW7Aoy/cDUq3EyS2i83i/m+ylawDdWJB+TC+s4KtOmF15u1BHU8kDhtrOqyVA3kbxoGOS/T/8GURDuWbKKPwUIf4PrwMoyFR9sFDx7z4gUlsx8AW8X2/rNpV57waFEeizrBBzLXp5zmcT4F0/zqMgkSRbv4P5oPCKD8ZVpJ/wdWwGxwILvii0QCJ/NT5fqUdRO7MmKvw5zU2fjzp7cHAseKcnwc8f8atOnWD32ABSFS+7Jhr8x9dAyvhm7h3LlvVScbFXNAIhYw3ifbXQ3uQYRF68S230Fe0jlZQfyg1jhl1tPyJxf4Ij/0TPc2+0UBAA7o7TmyX2uGnzkyNte5034JzoTRRx67/zmBYAo/DTZAoU/E4A2aDOj3951SGoKizUgcc/B8G3Qqbpx9Ed68q506gzu0cCIrfkcsvHaLg+xXIswy5jeMdYCLozvcTSCyv0ouAL9KnwugOWaJOMCgawXkmznVsR0BQRrGjej0N6jjV0SYaNAvxawuXgPKSEdEPDgX979NgAlwLQfH15vtJH2WJ9FaWroLHpQ+J/IKGiPwSz6P8dL4Tr0Ht+/pTmKCoPCUdC4YLhTtRMaGoVIgqsYa6fAtq/MIgP1EJNVjqBIwYDRSnAd0h9oDt4gcPZKsOaF/4AdSpTQGFUftVUht2BMfGgzqs/wMN3XeEoKix7w2XwnPwOyjdYIfLbmxk4AO2Z2kHaQNHyVGM7IPfz/B30CMC3d1K/DR74/Y7XD2emAbqpMojqus3oE5ZRtAQ2BXU8SkN1UeiOittddQSGU66C16tnQT5kP2dYXocpXk4BOU1dnaDZ0CGivJrDUHRKNAI0Lm6DCaB9H0YKP7d8AbI6PB1gbPcciwh/Xiq2zvljrkp4EycKrtfv/1xd910uMglo68x6NqTyH0cSK8S1atdzFU4MVMGw7egtNtAqCUt1LULR+V0oR4An4ajOrE/nuqzICT1MdVIrgZ+Irmb4TNQI98ZzgZ1erfBy6C70Q2wDNpAcfIBnjIW1MkuAhkD/SEoauzVEapz/wp2iW/bsk2Uc/DQ3b0MDBnzP0Nxnf27+N8Jr0JeHBkjbUCiPD+GJaCOR51heUX5jQKNLkg/MkhGgAxx6Up60530hdAHpLPFsDuosy9OFFZlPRwagvK4AYLyFDuqu9oijXKMhe8hKDounasjVh23AY0qbAtTQXGag+otP3WwFZGtCPwlrAMZTxXRG8FjonINh6dBbl1rS0F1Dor0eQaMh6+hPUhHMla7gMrwIvwCOgcKqzTk/gJsMGI0UGsakOU/odZyT37GF5HkyOQna1JMdQ1UcAJXC+rTE5oG6qUOsTuo05Fo2wiaQGuQbA3y86UjDqVTWifVluO641Rj3wz8tHAWiTqpxqCOVaIO9DhQnDYQFL+c28U91YEqrv7L6niUh+qRDr4oT9XDl8R931/bVqC7YG2Doji+buSvEQR1ZGnaiUtx6aoTVv1lxJQkKvMBIP2WJMpHxllnkF580TncA1rGPXZiq7prX+fZl+1xSHcSHd8h5iocpbgVt/TYKe7nbxRGYXV+FV+SmK6/r/L1B5VN5zECy0DGYnGi66gXbBE42AX3QmgAOofBa01laAPVKhX8H1VrWUziqamBHSnWb6lZtEqVSo3K7ErFNJHqtAbqeGP3Esp/Ho6CO+AHCHby7JYpvtFQZkATYDMN3M2ejIZkiB493AM6j0/Be1AR8Y2GisRJatja/h/JwjSS2hrQEJqGy0q7C0jtGmxeumx2dUcQtN43D2H2jAZSTwOnUKSPYACsgL1hLVREZhF4TEUimLAxDbzN77Qk6eIQ0lkKOo9fwNFQEVHcBysSwYQ1GqgNDYwm09NqI+NqyvMN0j2umtI2yaaoBmr7DilF1WKKZTRQIQ3U9v/IjDRU6HTVWmDd4eh5YlgkbPUJy3kx9TAaMBowGihVA8ZoKFU9KXMwbJ1s2OqTMheKKYjRgNGA0YDRgNGAjLuVEJZ5AJpAtgZKm9HOYSNh0kBtD6uGSZemLvVXA7X9PzIjDXXj2tM7wJ/BfnWjuGWWUhPI5kOvMkOaAEYDRgNGA0YDKaMBYzSkzKkosyBhG9IPW33KPIEmgNGA0YDRQF3XgDEa6s4ZDFsnG7b61J0ryZTUaMBowGjAaCD0GgjbPACt1fB76M+aqWCRBmr7WWxRQYzDaKAOa6C2/0dmpKHuXDxhmwewBNWvh93rzikwJTUaMBowGqjfGjBGQ906/2Eb0g9bferW1WRKazRgNGA0UEENGKOhggqr5eBh62TDVp9avjxM9kYDRgNGA0YDRgObNBC2eQD6At/cTdUzrjBroLafxYZZt6Zu9UcDtf0/MiMNdetaW0JxwzQPQB/w0edut6xbp8GU1mjAaMBooH5qwBgNde+8h2lIX4tWfQr7173TYEpsNGA0YDRQ/zRgjIa6d87DZDRI+2GrT927okyJjQaMBowGyqkBYzSUU1EpFCxsnWzY6pNCl4opitGA0YDRgNFAfdeADL0VEJZ5AI2piz5elQlGQqyB2p7AFWLVmqrVIw3U9v/IjDTUvYstbPMA1nEK9AbFnnXvVJgSGw1USAM2oTXx14jRQJ3VgDEa6uapC9uQftjqUzevqvpd6s5U/2Y4rxrV0J+028bTv5ztodWYl0naaKBaNGCMhmpRa7UnGrZONmz1qfYLwGSQdA3MJsVboDof++ktoU/iJX+Q7UFxt9kYDdQZDRijoc6cqs0K+jl73SEs8wBkNKhB1fCtEaMBaWAHGAoHQ3MIio6dBEeA34Y1xH0A7ArHQBr0hW1B0gt0l78TnAIVfRzWJh6vN9ugaE5Ob9Bjhww4FLaCROmNx7RET/YVbxjsk3BM9VIZT4V2Cce0Kx0MgUMgUT+t8BsEJ4DKVB7Zg0C7wI4g/ewFvkRwHAhd4h4qm3TdNb5vNvVIAzr5RuqeBsI2D+BXTsFqaF/3ToUpcTVoQAbxOfAVqI3KAl9a4PgbzAP9D/4BkgZwEQyERjAR9BXVq0Gi48/CcFC6utZug/KIOmd1wl9CS9Aogd92ylhpDZfAv0CGr4yVRDkWj9cSPGUoXwA50BH+D3yRMSCD4n04GnqDL+qsz4Ov4x4v+gfYyji6Cb6FlfBvUJnLEtVjFMhgkH52gztAIgPsdFA9JdpX+SpqeCmuEaMBo4Fa0sAD5Pv3Wsq7OrJVg35WdSRs0kwNDVRg1rc6vhGgjjQdtHx6UGQ49Ia94V7wxTcgmuNxZdzzRv8g2/EBt5xnQ6KhGgyvMLrLvkuOgByEWx25LzvieB4Utjjpgac640R5PcHjYvbVWfui+veGzuDXDafVE26FTpCon3vwC47YyWDwDSecpUqifs4gtPKQbAeXxVyFP4k6CRwyzurUQAX+R9VSDFmMRuqmBj6i2MU1RHWzNpsWeXqyrlbAlDtpGphBSn+A7mT7g4bYR4DkBNgLPgD5+48fcFqufuISdPt+S3xHfPsz261hfoJ/cFd34PvBzUFP3Ikd/rv4RRPC+LsaqQh2+r7/bN8R385huxMsAHX+KpfcMh6CohGGZSA99INM8MsnI+j/ICgzgzuluEvSj8r1CzSDxtAdtJKrEaMBo4E6pAE1lmpYg3cVdaj4fylqF3xKa7z/EsF41C0NVOAO6eyEml3IvobqJSNiv4U/aWyeDuz7owQaabg87u/7aVcdsDpViTq/R0EdblCC4X3/h3AEw+k/18Q/yHZHODOwH3S2ZeeioEfArccSemziy+04NIqyM5wOvuyGI2h0SD/B//157G8fD6xHEypPUJoGd3AfBMWVSY80fAOlEW7pJ1g+jTboMczdEMyfXSM1pYEK/I+qpUj60xmpmxr4lWL78wDm1c0qbFZq3c3omamQMWSk/mpAQ+rqmHQdqBPdAL+BZBbcAvLTsSicC1NBjzU0EjENNDowCjZCP5gEY+AoOB0cuANyQaKhd6Wnu2jdtXtwF+SDyqKwurvXyIN4CvS/OyKOOlR11irzf8GXM3D8098JbPX45FW4E5RuK3gDVoL+1x3gGpBxshZ6gkZevgQZRSqb8pJbdVgCkvtAhkMeqOwyGN4Dpe3LKThOgidB+vFlHI7D4VSQflS24HGNNmwDX4D0Y8RowGigjmngWcp7Vh0rc2nFfY2DauCNhFADlbhDUsdVklTmTlfGQFWkonnqpkyjJGVJSemW5O+nV5p+FKas+H46/rY8+rm9Eun66ZttEjRQif9REnLdlERZF92mkMaVihr4iEIdkIoFq2SZVJ/9KxnXRAufBtxSqlTRO91BpNUDTi4lzbIOVTTPAhIMjjqUlH5J6Zbk76dTmn4Upqz4fjraDgRfP4nGRgbHTgONxrSBHcCI0YDRQB3UQBfKPL8OlrukIvfmgAwHIyHUQC3fIamjawM7hVC1yaiSr5+di0lMRkSbAHo8Y6SWNFDL/6NaqrXJNlka0EjRn6B5AGEQTb5aAw3CUBlTh801YBq7zfVh9owGKqOB2v4fmccTlTlrqRNHw5OfQlgeUaynLnNAE76MGA0YDRgNGA2kmAaM0ZBiJ6QSxQnbPICwzdOoxCk1UYwGjAaMBlJTA8ZoSM3zUpFSha2TDVt9KnIuTVijAaMBowGjAaOBatVA2OYB6D3wZaDJV0ZCpIHafhYbIlWaqtRjDdT2/8iMNNT9iy9s8wB+45RogRstbmPEaMBowGjAaCCFNGCMhhQ6GVUoStiG9MNWnyqcWhPVaMBowGggdTRgjIbUORdVKUnYOtmw1acq59bENRowGjAaSBkNmOfGKXMqqlQQzQOYBVqvoSKrwFUp02qM3Jm0J8Lu1ZiHSbqGNcCzWM1V2aKGszXZGQ2ETQPL169fv2XYKmXqU/MaWECWHWs+22rJUcbscti6WlI3iRoNGA0YDRgNVEoDkUrFMpFSUQP6wp9GGb5KxcJVokwHE0d3pvMqEddEMRowGjAaMBqoBg0Yo6EalFpLSW5FvvrY0+Rayj/Z2WoN/Pagz/4uTnbiJj2jAaMBowGjAaOB+qwBzQP4NkQKeIa6LIJ1IaqTqYrRgNGA0UCd1oB5e6JOn77NCq9vNmiSWVjmAWj9CY02aPEqI0YDRgNGA0YDRgNGA0nWwCuk1z/JadZWcjJo9QXPMLwNUls6NPkaDRgNGA0kVQNmpCGp6qz1xMK0voG+4NkXNtS6Vk0BjAaMBowGjAaMBkKoAb1x8EnI6nVZyOpjqmM0YDRgNFBnNZDyizuZBWEqdm3l5+db6enpFYtkQhsNGA0YDRgN1BUNmMWdSjtTtf1Fr9LKZo4ZDRgNGA0YDRgN1KQGartPNHMaavJsm7yMBowGjAaMBowG6rAGjNFQh0+eKbrRgNGA0YDRgNFATWrAGA01qW2Tl9GA0YDRgNGA0UAd1oAxGurwyTNFNxowGjAaMBowGqhJDRijoSa1bfIyGjAaMBowGjAaqMMaMEZDHT55puhGA0YDRgNGA0YDNakBYzTUpLZNXkYDRgNGA0YDRgN1WAPGaKjDJ88U3WjAaMBowGjAaKAmNWCMhprUtsnLaMBowGjAaMBooA5rwBgNdfjkmaIbDSRRA1p7vFUS06vppNLIUJ+GN2I0UJwG9MmENpBR3EHjV34NhMlo6ES19RnlIDPZH1JOdajBvLmcYYPBXmRnTtCjGtyXk6bq1SuJaX9FWicVk97e+M0vxj/VvA6iQH3ihbqDrUh1URn/mcRCqpO8KEnpqWw9k5RWspNRHcsyCPRV1DHQMtmZB9K7HvdDgf2KOvVBuWD75LvvLyOhZhy/ooww23B8TRlh6tLhbSns+iQVOEI6n8OncE6S0qy3yYTJaPBP4mc4RoAawUYwGo6BsmQ8ASpzQeni3rmsxOvQ8bmU9ZQ6UN5LKGOY9F4Zle9LpOIMv4qmJYNbBsM7FY1YQ+HvJJ/MMvKS0fAo3FpGuNo+/AsF0F1vkKvKKFR3jteF/2QZ1ai1wzuS826wPfyn1koRkozDajTcwvm5CXQnWgDXgkRDUyPgbXgLroN06A+doSU8DbJMdZE9BdNgEhwNlRE17M+A0rkbgkPAyndy/NhItm3Al/NwvAlPgO4igrIHO2NB9VDdVAfJUFA6J4Pqtx9UVLYkgsrVBP4HwcZad3y9QXIivBJH4X05CIfu+FS2myAYn90iORXXq6A0gh2f7sQPg4nxY8Xp/QiO7QVnwZEgaQmPgPR8A+gcSpS/yvEuPAcdoDjZGs97QfGnwADwZXcc0oXOx42g60hSkv8uHHsSlNbt0BgSRWmonO/A86DrT7I9/APU+UkHqtf+8AIoPV1LylcG8RWg+twCko4wChTuTmgKkhPgFNComNJNlKvx0DnzpR0OXXeqb/AclpT+cYQbDA+B9Hw2dAHlJ3qCROFUjpHwBlwIvtyBQ/n6ov22oDo2hAdgO0gDXfNvw2joBr7oWuoPW/gege3/4T4CsuA1CF5zu7H/FEhv90Bz8GUYDl2nT4NuEHzR9SW96X+m/6L+k1UVnftHQeW4D/xyXI97J9D1KZH7vyAd6HzqWvBlKI6poPMZvNb7sK/rScfOAV+64lDdlOfd0AISpQket4Pq+gL0Al90PvV/UXzpzr/mrsH9N5BupO8DQOdG5/0x2BIkf4d+MAFUPpWzODkEz5dA8c8PBND/JnjuWgaOyany3A/poP/wziDdavRW53VHaAOPg+rwAPhlOxj3aaD/ko7pfOua1HU3GQ4CI6mmgQp80asTZffgoYQ6fMP+urif/nR5oIsoCxRef6AzYSVsBF0c6mgWwA8wAmZBPvgXE84i+QiXn36RZ9xxCFvF+w70Z9sAc6Ax7ANReAX0Z1Uab4FEZVLZpoMaB5VZ+/qzqmFdC1/Dv2A96IKX6IJXmr/AV6ALvCTR8ZOKObg3fvPj/u+zPTHubsB2GWwPQ0H12Bf2i7v7spXe/oDesBuoPjdAolyAxyw4EHqDdH0KSORWPOnnNJDOEhsClWEaSG87wB2g8zcYFE/lPxUk+oPLWGgPw+FnaA2J8jEeahjbgOqsa2EbaAW/wnmwK0yEf0JJ/iqr8jgXdof/wCSQqJyKK3kGxkAHGAJLQPmpEVSddd6VRhv4//bOAs6qouHDH2WQKgImLKECYncvYmG8ioUtYgd2oa+C3f3aQdjdIiaIqGCLAirKUoIothiAfM9z94wernd7gV12/r/fc8/MnHPmzPwn7917YQbsALb99TAM6sKZYD9YH7z3azgGHAt3wBBQ54B5HAfmk1ZtIt9BiySxCUfLcixY34fBflZc/r05b/76tg3Yl18A+9J/4UNQ9oWfwes6wdtgedUnsGkmVPhi3PvXAPu799SHW+FR6AD2X8u6MgQ9QCD0pZDmcSS8AZuDeelxS2gK0+AE0Dd9fxNqwV7wGViuXeF7CPPLDYRdLL2nG1iO1lCcXITMw34d2De5wbp9AaeA7XwpDAXVCyyHfXtJGA+nQRvoD7a17fMX3AVrgQuj/irrPAlsm7XB/nMCKMeK9bTsA+F2yNaFJPQDr9kPvoG6YNlHwTpgO70G54N6Cd6FjeF0sN63gWV7Eq4B9TKMgQ3A8tmHV4flYSYo89Df7cD7h8CpoD4Gy2HZLKOkVY+I4965Kx9WgN/hbjgabP+JcCbYllfAR2D9DgX73sGwGZiHfcjwUeC8sMBVhjVxgZetSjywDAbZ4HMhDOpQ/uEEHEy1oQnYIZtDT/B6B4R6DSZnQoUvTkTi9QPAax1w2TL/X7MTk/ggjna6ZZP4YRzNx2NdcKAtDVuAk8LnoCzL97CEEeSk4H3rwcVJeDWO6lr4A7zWgeh120NJKs2m4RAyeTzJyInluSTswDkB8hKc6DzXEH6GU6ENNIJcep/ErqkT5j0iiY/j2Dl1biJhfcrWQyT0SBIv4nhrEvbghKtPzWA2eH9eguV0ssiWE5Ntoo+284zkeDDHZyDIycxri0o/nHNOynkJ7Tg6SS0HlvMysM0tl8/JS3AB0lP7m/2pHig9Df1uGcLm/yWoneDVTKhwQ/BIEvZgXayDdT8HHoBcyiPRTUOQi8LgEOHYAjaF46Co/Htz7n4IepdA9yTiGPohCZ/F8c4k7GEHCO3+CWGfE2R8wyTyI8cVoAHMgo0gL+FhjqdDUF8CbnKyNZKEPVKJHxDeCg6DdPvWJu5isAE8Cz0g6HICzi/2Edt0c8hLuI+jPhenLTnpfbZZwHZXe4GLVV5Ca462SwfYAhyvamd4MxMqfLFP5IP9aw4sCWpFcCwqy3YF5CXszXEMqAlgvTqC99pvsnUlCZa3CywOjUCtBHkGkOHbIbTvS4T1Vtnf50KYBw8m/Ciol2G/TKjwxXI6dh1nMwuTMvPvNYTzEmzHMFd+Qfgq6ASWP4wbgn9rFUJfJbGmHC2LR6UXr2VC/7yYt3U9FCxf0CACvZKIPpiPxwWqMqyJ86VcuTrIfHnQQszUzjcD3DhsCNdCSwiD0Ekil+zwx8Jv4OBVRV1bePbfr21ImgjfJqfCM/OIO9gdHNvBaGgIYZA0I2wnd4JRYYAYXtkXNLbw8Pfrin+HKu+LmU7IThiW9SDoB8oJ4kDYzUgiB+8v8B84Bc6DL8GF8BVIyzZxsgoqINAiRDgGv036E0rju20c9AcBJ3Y9mQPXQ1rmma32JPQHx8Q7UCvBxWoyBE0lIJ0hV3pX0m334BXBzCTf2EAi8/wLnAjTCu1tf5mVnJjN0b64O0wDPbVs2cr21PssX/A1TJrZ9zUhwXYLsmyTQoTj1wk7cUy3WXb+36Tu8VxoQ/1Pl3d86jrLZLlzKX1POO/CaPqlISE5hnFj9GdomaRnH0KZTLcPmJfPL4Ag28Uxq296MQWCLLv9xPG5GFwAab2djhQRtp92znFuJdJ8ZrrffEjchTAty5Tud9ZpCOiNPvwGyjFQNxMqHK9rEV4/iXuwH6ld4Ux4E8yrNzwAaZ1H5Fy4FSzjADgRXKBvgHVhHMwF54Eg+7GyP6jvCw+ZMan3QQUhwFG/107FDerNBpBOD8/pRrplfh3M/79wLxQn2z7MF8sTnpB1cQFx66mK69eeT9fD+CKv0KkW1YruSMVaw8Ckgrdx/Anc8TrICsBJQtnhQwdwcDlILgI74alwBXhNWTSei7cAn+cAWg/UOHBh3RkcCE4Ob0FTUE6mG0Ij+BlWhyDPqQ4wFjYGB4HPCvo9BCp4/IX7n4IjwEG7JygXzevgPiNoVXCQLQX1wXotASfDleCkkpb32y6jk0TDThYVUa62mUaGbjicWH5IMvcdRJhwkqTMpOSEaFuMAe8Jk4qL5uYQZFkPgQLIlW7drNe2oOrAXjDBSKJQLtPtF6ozeI2LRLouhxJfH1aBX8Hy3wnZ8rlbpxId2ytD8DWdZ+qyTD1ttyDrG8puWks4EorL375RVP6cmkcrpWKtCQdfHIeLJedqcXTMBIW8pycJ+3P8KglvydGyBVmX4GlIC8eQT4h79F7HUJBt3wr0bWIS5pBR2Jib/2w4BApA2RfSC0wmsQwvlmMSdE7d053wp7BOKs32sV2DmhFw0bw8JOQ4mvfzcHFyzo2HmwjHaFvYB+wv+8HN8ACkZf+7Gk6D9vAiPAlHgmNpF9DbCyF4RPDvudVwcUr3iTZcWJB1seV/Dc5L0pfj6Fxqf1kV9oU6sDc4x98LxSndD8y7ddbFxm3/dpC+Nuuymhl1gCxq6kGFxsMEeBZs/P+CciedBy7Yd4MdojEoFxUXvofhD3AS2x7OBq9XLuK55ETvM9NsQdyNhh17BNwP18MoeBAsizoWbgIn3pD/HYQbggPFa130gvoRmAmPwiXgAL4KLG+Q9VLngOGORnIon7SDU2yY4xqfZz7BFy9xYrkAtoGN4BFwEqoFXn8AOBE4gXwG2bqVBL3ZCrYGJzPTyqJfubgLdCjmJhdnNz2WaU1wUnkInCzTmkVE/1aGVeA6aABLgpOjE9TR0AlsQ+tZVLo++Sz7jGW7Mgn/yTFoBgE9s1z6tgd4X33I1m8kWJaW4OR9PlgupQdtYEfw/s2hF/j8W8C+NgaK01ec9Bnmr56BNeBYCPWtTbi8+XPrPLJvdIdNwf57MyjLeSTo9WWQ9sJ67gf14B64C9aBXeFRaAhBqxP4IERKcXyca3zmSaBvN8J4MA/L5vjvDLvA4aB+g/5wB7gh3hkegyag+kC+gTJI35uB99pvbGdxLrL+jqfd4AVYHk4F2+dayO7PJM0jx9aJ0A2sox46duyTV4B1t/1bwaeQLfuX/cnnrQCLw5egD8uA9+lBTwh9k2Cp1ZcrN4XuCQM4pnUbkeNgT7BvOndvB45b5w69sAx58BmURc9xsXU/C/TmSrBewyEqhwN1cqRVqaR69er1nTVr1nmlKJQTSlMYAwVg5+kPx8DXoF6D5rAa3ATj4Gcw/Qvw3FywU34IraEh2KFqwefgPWk5gCdBQRYvEx8Bz4MDywXpQTgCfoF3QdlRR4ODUtlZR4KDsg1MhvNhDgwCnz8YnERWBSe9XvAH+Bzr8zT8CUuB6c+B6WmtQMRJKi/FD4Q/BSdsy68mwvJwM3wD6gP4CqzLptAfboPfYRgcAPuBfp0JliGtd4j4rCPBSfdyeAjUijAUfjKCsuOFqYUT+wZEnFCnwFSw7dXSYJt/AnqRBz6rBZwAH0Fa5mF7Hw47wKPwOuhZyGMfwruCvlwG3mPeudKfIN1r9wXLdgzMBMvlRibkaRtaLtviZHgPFoMm8AIoy7oUeJ3t3TuJD+doH7FOHcBF/SnYC5xcPwYn2j/B+20707Jlf28P9eB9sA3NZ2/YDWyLS0Avist/BudHgbI+I8FnOsc0h2dhC/gWrHdXuAEeAKXfm8LuMAS8P/STrwhvApa/HywPR4ALxWngtWoxsG1OAeuRlveMAJ+vQtyxa3v5XOv8Kdhef8A40OOjoS3cBLan9RwEy4LlcJyeCW+AyocJUABpLU6kPgxOJyZh28l+tx0cDN+Bbf4j6GNjWAceg8fBtrGt9agPKPvCM5lQ4adl1vFpGA/vQk8wf59/EfwFz4N195m/QC/4GdJyTK9GGpp5AAA4t0lEQVQCh0MnOAPsq9Z3AzDdvPqCbf8SWJYPwP5eG2zzJ2EuNAT9td18rn3DMujjcTAG7DfLgeXXy7ehB+wA5n8+hPLrRQ/4DY6F7PLXI20ZsK7mG3whmBkf+r4j7AeTwPr8Co1gJuixsk72Qedk1QqeAsuxwFSGNXGBlalKPWhhf+mjSpkRCxMdmD8OuGkYNn+ynifXs4hdO09K5UZc9K+v3CxjbvPZATfhu8znZyxS2S/sNdEdYFR0IDpQsx0YS/V9J+e70Ooq30EeAedV1wrEckcHogOV4MDC3lVVQhViFtGB6uBA+Ai3OpQ1Vxnrktg014mYFh1YlBxY2Gti/KRhUepNsS7RgfI7MItb/Tt6ddVsCj6juhY+ljs6UF0ciJuG6tJSsZzRgehAdCA6EB1YyA4sapuGvJSf1xFunopX1eAVFGzlChTOj2T9Vu/8kN949mNfdSm0zIQW7Ivfxr6ylI/029Z+Q11dAG0zoer34hf69qikYts/7gC/ub50OfP0VwINynlvRW7bhJstt32vvHKOC/12G8I9y5tRNbuvN+Vdu5LKvB75+IuN+fkl1lDUWgT8VYLy1zb+cqWiWpIMboHnwF97lEdLcFOb8ty4qN2zKG0a/CmTP4cJ6k7An8xUZVm+XvBtOQvpl78+gTnlvL+k28ZwQViE/UmUC9CC1lI8cK9SPnQU14U235Vwddg05qpaDxIXy3WiHGn+bC8ProEfoTzyJ2duHBa0LPMHcF8FHnwh9/pTOtUBNsuEFv2XrlTRTX9l6HIy+RwGVEZmJeRxNucPTa5ZheNWJVxfmtN7ctG64Bu0CaW5Icc1g0gLm5kcp2tO0qK0aXAH6Y4yH1xMlV/u2gn8DW5Y/Ahm5OSxL5T0Ln8ZrtkDnHCawFqgOoLv3PKhE6hVwc3KluCOWXl0x+zvrM3HgRC0HoHR4C7WhXE1KEp5nPAd6ObJBS4q+4N1bZ2k2Z6eNz09YTQl3h58x/UfUG6y9GZn0KdsOcgagmUPC5jP2h52AcusvMZ3NKvDdhC81wMn6zwIakMglNU043kGEln/fWBFWBP0O8h2sP4bh4Sso+2ix5uC/UDpx7awK4Q0ghnp/QFgGYqT/cNJJx/ML8h6269sU/0Nsg/os3XxPn3W366gJ0GebwH6q2/pvNcn/jaoxWEH6JaEOWTUjlfv3wjs+7lkOdaAyfAd/AW2peXeHfQrLb3Q/90gbL46Ebaf63sD0Dc3ckHGbSfLaTnagv0q9Cnb5UCwvGl5ne3pOKyVPpGE7X9+ymTZv03S7NOO2XwIfvncDcGy7wzWL6gZAX32nAT5bMdIOs1z9jvz7wy1Ici2tk76ku2Z/hwEtntRKu5+y34w5EN45vKELZvlDOOBYGYBTXvrdV7TEZw/jBclr/E5+pFWHhHvtW+G5xP8W7afvod2MK6v9h/9Uj7XcnaBOqA82mfsR15rP1V54LgI9xL8W44jy+ninO4vecRtr1UgrWWJWPZtIDw3fd6w857tNgomwXSw/I4n2zlb1s9n2f+CHKPOK9bFvpo9BsLYcE70nPXwmcYtl8+xztl9h6SoSnegDN8UvZSHOyn2AxtrGnyYxIdzFGUjPgVD4UooADt1LtlZvoCrYRg8CbeBehsegMGwL1wI78P58BrcB6o9fAuPw6Vgp90W1GlgGZ+Fy+E7cNBka3sSzLsPWI6boSW8A2PhDLBDW8eH4SKYAvmgToDn4C24BRwQ1usGuAtMrwdpnU1kLtwNS8Nn8An0hyHwAeilg+RLGAPm48AYBG/C9WA5DgdlHfUpyHjfJHI8xwLwHj2ZDPqvHz+Bfuu95U7nQTQjvZwN90Nz+Ags0wB4GSx7qKN9ZCRcAZ9DKB/BeeRE4X36/hKENl2LsHXWvxvhK2gDyv7wItyThF/gaL+xr02E7qBegdfBdI8PgmoNMzKhwr6j7zeD7aYvDUA9Bba15ToWcmlnEkeBfecUaAHWR4/McwLorzoZ7EuXwLMwGWxL2+V38Fkrw3uQD0HGbadWYFt/DPbLZvA/8PwV8CnYD9U+YL3Oh1fhCcjWuST8CNZza9C36XA9DAHHQX1wQZsK1tPnWscgF+TRYPpe0Auss/feCd/DtqCc1EP+lmk4NICmUAA+9ybwWdZVXQWOA+s3Bk6DbC1DQgF4/40wDfKgNjwP9s1L4CMIfcD+aFtbjnvhO7A/DYQ3YBCoI0Bfh4Bl+BpsC/Ua2P7qMjB/r9GP3qB2gXFwAdhv7UvZOp2Eb+E56ApeY3/Sw2NgV9C3G8BzI6BRguX2uttA3/qD/d566H1LSGttIpZTT124e8JEsC1uB/P7DyjraR/VV/vPm2B/yNZJJPwMT4J+dIPxcCk8AwNA1YL7YQicB5bjfKgDeqO39mc1A/IMoCXA8dEEtgbLYXkfh9bwPgyEa8C5ZjmokMqwJlboOdX25jIY1JhKzklV1MG5dxK3YX+FFcEJayTYSdRaYIcOcdOU99gp1zOC7JDfw+GwOPwBB4Hy3K3J0Xge2LHUgeDADB36BMIPg3oIHoPw7KGEd4Rs2VkvShKX5nhLEnYgnZKEHTyhU5vUE8Lk4oTzLNQFn+UE54QR5MLmhJCW182FBkmik/xhSdh8voEOoD+zoQWofWEk1DaC2sMv0AQuhwshyHhf8Bk/QZhEliXsQHdi6AizYHlQW8GnmdC/X34jyXuVg/64TKiwLFMIrwMObNujHqiV4EeobyRLljV4an+4E6y7k/UOEKS39is9s92PBeUk5YQbvOlHuBfojc/sBqoR/A7NwT77PCgnoNDHjD8I+xlA9tnTM6HiX4LHXnUd2E+CjiLweBK5kmPbcILjl7BZErcOrZLwexzzk7AH47aT5+fCaqA2hkmgb6oZWOelYSCcDcqxZB/Wk2zZzuuC56aDeQbZZ08C28/n2k9yyfqemZzQ+/fBdlIXwE1g/tMg1JdgZuycytF2to6LgbJ/69M64D1hfCxD+AewnmltR8RF0HqqfaAdrAA3QChLe8Ler5xjHKPh3JuELwFl/nPAMtsPP4E6oMx7aCb0z6ahE3G9a5ikL8Xxe3A83QwXgzIP2yHU07SgdwlsmURe4nhZOMFxAnRJxZ8gbNvap20X208dD19AaOcXCetlti4l4fwksSfHsRDqZzv2B/UR7J4JFb44NnxGtpYk4XewPM4NzlurgjLfSbAy5IFjIJRvG8IjQNlvBmRChW33dRL2sDHYVuoMsLzOdephCP3cuPmXZsx6bZEqw5pYZB4VOeEEuCjLwarsNA4UB85GsCK8AkEtCJg2OSRw3A4mggNGzYQZMBLWAs8NBDULfNaN0BTshA4QtSHcCt6v7LT1M6HCj3MPIOzgqgVrwCjI1h0kPAlbw/lwFKj14e5MqHBn7rOCXCSdyJTpLj6zwU7upLUHhEGXR9jJvCQFP83HeujnX6AXYSCtS/jlJJ1DZhA5aXU0UoQ85/3mo76FDzOhwhfPTU3iHn1uaRTKaxnNI7S/g9pFJ2gJAk7aLg5p6e2zYPqFcCioj2B/OBDsO5tBX2gLtcH2VmvDIxC8MX4XrAb2pcdB/Qz20SXBtnobvMa+Og46g9Kn16AlLA7XQllkftb91eSmxhxdhNRFcADYt9cE61UXyqKfuPjT5AbrUR8GJXEPLkid4E54FHrAc9APbKOiZH1to7dSFwwmvB44Mf8Ko6E0sl853tRXYN6O/UYwHILMX79ugd/A/m6fsdzj4XCwDZ6BoHoE9O+lkMDRPH+B6fAihPvnEL4J+kCHhLTf9rFQzu8IfwLqR7CPhWuHEDYv5bNuz4T+ebEO+v70P0mZDbPl7A9PwT5gO/WDP6Ek2T9Vc3Dz86qRRPrm3BkUxrF1GAuhna2HfpWkUVwQ6md7OefZr+xHp0EvUK3AMZStdUj4Ahxj3ZOTvZOjB9twaSgA2/hOWA5aw3ugNoARmVBheGQS9rAhhLjXXQXWzXG2G9hW+qo6wiuZUDV+CR2vGleh2KKHzpa+yAXPwds3nUh4WlbcwTA5lWanbAEfw1GQnhicvJeFi2AinAEuUMqO5POCNiLgQGiWEDrjKsT/gEmQ1lJEzNPz+4M73mPhGXCysWP7LK8bD0E+x4nUAeGzQse2HsOhLyg3K41gnJESlMtPb0lPNLOIO8Gn5cD8LUlwEAV53UxwsDcIickxHS/quVm3/Cua6z7bX8+Py7p6ala8MfEZ0B72guthGdDHR+FI+BDUZ2B4D9Bbn6Fs+0cyocINQWivPUn7NEn34DMsq/3Ne64E28k8z4Eg29h22h6GQtp3oiXKct0MLhRBPrcuWK+XwLqdDsOgKGW3YbjOtg/yWR/AoSEhOU7jaHuvCFvALvAGrAnWLZfMtx743LDo2KfCIpF+LsnFyvpmy/vrJITzIf9fSN8MbLud4BJwQZkOH8MhkNbX6QjhX2FzCPdfRLgNOH6HgPGrwEXtbQjSv7RCudJphi130JIEwjgLaeYzBrLLafkdeyuB5bNu9qlN4CMoTqHfebRNbBvnLqVvoQxzCafLnQ57bWmU6x77gHmfANYjSK+z5XgKvjqmHgPbUFnWy+FzeBY+gStgAvSHEaDM43+ZUOGmZWQS9rAtDE7iXndaEl6R4ww4JYl7yJ6jU6eqTzA9+KtPqXOX1MFRCxbLffrvVHd6W4ODtABWgRfBTpiWE4IDyIWiAdwI74Od2M7xFgStQ+BucLB5bQ94FxxM7ujXAGWn7Q63w/rggm+51YYQOncmIXk5iOO94ADtBw+DE+zaMB5+gV/hGzAPlQfHwM3gc96BMNk6QFaFb6EAXOiuBieQtPTDui6RTixF+FWu6QZNk2t35Oii9An8AKuBqg9dMqHCc9ZBb5QD0TqWVXpZUnlt/03BuhXA0vAmWMa0rMOT4HX3w0CwLS3Xh/AMTIKzYRTYPtltaD8JbbouYb23nqbbBouDOgduyYQKP871nnGQB6Gf6tUdEO4PExpJmU1jvoESZN1tDzcnBWDfuhSaQVvoDYNgBXBc1AGV9vV74qtlUgsXQq/LpSEk2vfMowBc0Cyz7fMQ7AcvwckwFVpAUZrCCSfyA5ILGnG0r9jXSlJp+vA0MvkS9EM1hJC//eAFGANu5u6HlWAoOAatTwHUg5HgvWntSsR6hvvvI+z968FoME/v2wGy+yBJJaorV7gYqYPh+Uzon5fXCHaC+lAAzvk+Tw/7wVGgj6eBHiwHpZXj2bFwSHKDz9gXStMuyS3/OswmRU+Lk5vFN2A7KICJcAfsBtlyrIUxOI7w6uD19qcz4EjwmevAbWCbtIOdwTlcn4xbT+W4nZQJFV5ju5m/vi0O40F5jX3BNi2A9vAE/AHVWuXppFW1wi5678NUcIIuSoM4MRDsHGMhDw4DJ5e0XifidW4EHEx2VAeb2hAuy4QKX27iYIczv+/ADmnncIGx8zixDofl4Wj4AvaH0JkJ/mvBMU3dDbuAnfYnsGM6OOykK8OD0B0OByekAlgJnIwt71nJkUNG7/H6AHwMDvpvwYGeS8NIdKHbPNfJItJeJP1e0F8H0LKwF8yCe0CvP4Da4GKr/oK9YQDcAG/BFMhuE5KKlW1mnl2KucqJ4ALQh0+gHRwPv0JajxDZA/RJj5pAN7AN/gu2p3X6DMJE4AR1CSjbwEn0UyPIc6G9DZu/cX3Qj/OgI7goTwN1LXjua7D/+fy54P0XQZATmRN1HdDLonQpJ2yDcTAd7EvmORXuBr2ZCD7rNTDfV2AY2Jd2Ass0EA6C2WB6Ltn+Z8Kb4ILZFk6FH+ByeAIOhWYwBN6A4nQAJ+3rx4HjSP/ugxWhOJmvda4Pk4q50PwfgmOgBVi+u0GPjoKxMAPsB45H+/ZJMBRs4zZgfb+BtJ4n4pgP9y9F2Pt/hlNgBOij/fFXaAVl0S9c7Hj5DeyHu0FaltP+bTt+Dq3hv2CfuhqegX1gGbAML0NZdDAXPwqHgW35HNwJDaA8si7mtyQ4nxelnpx4DPYE2/YDsL2y5Vi5Pkm0vzim7Zt14HU4HdR1MBi+TJjG8U+oCz+C7bQu2J4XwxHwLOj/h7AtjIQg2/dkcBwVQENw/PwOUfPTgXJ86cPGKY28rrgB2oXzq6QysgN3TcWzg41IcLJJy8nm3iTBya12+mQZww7qFbLusQ4OriAnuDyoFRKKOTblnHmWpNL6mZ2P9+VlJxK3bKYvBkEO4CPBo3KgfgttjZRRpS2vvuVBSW3iJL8SZCuPhHrZiaWIW28n+KWhMbgIFSevW7a4C5Jz/UpxTbjEtl8uRFJH05un4ulg2lcn6Zbpk8WEl+BcHoS2DZeGfmD9yiLHrGOtLLK8Pq80sl62S7b0xU16tsKYs88WJxfUXPfbt8pan/AcF657wL5c0ubJfpcHucqpp7Z9RaRvJfXl0uZvfbL7S1H3Wm/HaFlk30/353Cv82EuH2zjdN0cj7n6SMgnfcy1LqTPlzlcjjWxzM+o1jcsRIMuxrg3oAfcAa9DSQsMl8yju4idNE9KjBTlgO8w9PlgeAQGwaKo9amUn05Uppzs9qnMDGNe1cKBsGmoFoWNhawcBxbimlg5FZjfuSxEg9zp+lFfH+gO7tTLqq7ckOtdalnzqQnX+3HmoaDfB0J53sVzW5VXa0q4TZUvZSxgdXBgVQq5RXUoaCxj5TmwENfEyqvE/MwpGjQ/3Y15RweiA9GB6EB1cmBhr4ll/bi9OnkbyxodiA5EB6ID0YHoQCU6UNM2DaX9MlQlWlyurPwCZq4vTZUrs3hTdKCMDvjFr82LuMc/t61WxLmYXDEH1uR2vzBZ1VVVy6l3/kLBP9tE1VQHKvGjmH542Kqa+Pg/yhl+CrQwivwgD/Ub55WlB8iovD/BqqwyxHxK74DfhP+xiMtPJN0v+JZX13JjuzLc7M8rby/D9QvrUn9dUR5fLuO+jkmh/fLvHkm4Kh+ep3B7V2IB88nr1Arm53fQCuA52B0WWVXimlguj2rKJw1OUjvChHK5VLNusk9U5oTgpzt+kbS8coI6E8KnRP4kyi9LHgbqIDDu4lIa+S5pQUo/e8OLOR7qLx4suxwLJf1sjkv+lj6U5fq/b6xgwEW/ZwXy2It7yzLvtOH6DSrwvAV1aysetHE5HrYn99Qtx32L0i1+mbO047eoejselgM/aXisqItieg1woIy7KifRwfBugp2xE0yF3yH8mwn++wlvw6dwP4TfSe9P+CJ4GoaCvxe+DsbCx3AIlKTNuOB1mAKfw+mg1gGfNQC+AZ8fJkPf1fsuZRJ47/MQ7iP4t+oRugK+BMt+Gih/2eFk7v0FcDG481bWw3eH42AyhB29v5/vD/78bwycD+pRmAsF0ByGwYUwFXaGN2EVCDLuxK52A/89C5/lwPWngP58MuTXgvBh8An43CfBgV6czs5xMjvNRbe0Ksu1pc2zNNfleq7tflJyczOOtsF6Sbykw0ZcsENJF5Vw3n/34wmYBvZvN2CqJfwMN8IEsI1DuXoSvhzU0nAveI1tegAE5RN4C+wLjknz/B/MBvvhWuA7wlFg/3sR0v2KaEZTeP0DXi+MZn554tixzK/CGkl6+mC53gDL79gdCRuDcqxcA44f83gFWoP6D3wIludlaA+qqPTCs4Wv4zn8CSOSxM5J2GcMhbWT9PThKiKzwDpuCIPgNrAMzhFXQy1Q+q8HE8AxqX+51I3Ed+ErGA37Q9DxBKybbaI3dUEVlX4k57y+APpBQ1DOT3tnQoX1sn6Wazism6TbtrfCB2A7OR9dCuanJ9ZBfzeB7+FH8FMXdTJ8DvoyEJYCZX6XgOmnQ9DiBCbCHLAdVgXztQ/4LI+h/W3L2+E9sAzVTmVcE6td/Spc4DIa5EA4IXloV44uTOpaODcTKlz43yHcHGqDnfJKUA5YJ7/NYHl4AJzo6sGy4MBoA0XJQWgn3Ta5oCNHJ0nvtRP/BXZadQG8lAkVTsIu1j6nA3wL6UFBNKMzeX0BGsDSYP26wKXwHDiom8Br8F9QX8AAWAycYP8A6+Zk8izogYPSyc7BZnwuuKApB+E9sBosA5NhdQgy7rl24ES3Jqjr4E6oBeZnmc3TxUjvlZOmE0RxOjvHyey0PqlrzPsiuAL0JV3WHsRfBa+XQyGtw4l4jxyWOmEdbI+joT1cBj6jEaiV4WKwH5muv9lKlzGc04+TQoSjzwl9sQ5h29D7zoMdIagTgdvhHgh1WS6c5Lhzkm6fPwvqQS4NI9Hy+lzbcAp0hpYwF04EtSdMBPMx7S5Qj4Pltc/owZfg5Gw/sS/kg7LfPp8JFT7Dfqamg/1dnQH6mq3NSPggSVyJowvMVkn8QI5fgT6m5Xiz/KcliXpnP10MesGLsATUhXvhWlBes1YmVNguVyfhotKT05nDBryOThJsi+/Asan2gWkQ+otpQV8QCGNmEOGh4Ni2H3uP9W8MU2EHUB5tjyWMpNSC8A8QPLUfWA61E3wKK4A+2B72vaLS9Sxcb/veCPY55b17QwOwz+wCyvoatz1sm5mQD5bnEBgOnjO/2+BOUOfAVZlQYb5uLGxr28dnPgbK/vYW2Ffto2npuXOLWgq+hVAufZgB9ot94TfYGjpCtVMZ18RqV78KF7iMBjlJOAj3BweGnU7ZWbtmQoW77E2SsIf/gANVvQ97ZUKF73p+J+wAzU8YwrEbFKfFOemg3xSOAQdOa/CZli3IyeDDJOLgNB50C4HTQyR1dMCEepjcFKyj928OQTsSCHn7zI3DCY4FsAbkgxPMxbAluEgpB7QTroNbjYctMqHCFyfQ1bPiDuLj4e5UuhNaE6gF5ucEUw8mwYOwDzSGknR2jgss88EpwoTjpTeBz1LWyXPp5/TxRA71JC0/ld6F8AGpuEHL7SJsvo3A+sitEPxqmMS9Jq1cz/Wek9IXEc5VXy+5MOu6jYjbN0tSRy7YL8dFzUj7A+yvQRcR0D8n5Blg2wXZj3zmiXAXWPbZ4IYiP+EhjpeBaa9AkB65kVBTYNVMqPDd3nOEDwLLk0uOiw+SE4dzfCLroo+Jb5+V5uLgApJug0+IOybVkqAvu8LT0B+U42sw9IDmEFRUejjvcQMYnSTYN61XWu8R2SWdkIT1dc0kPIij9wYZ3wt2g3GQn6IgCXOYR7anZd8aesMcULfBGZlQ4YtjQh+KSh/IuX6Qn2AbfQfqedgbdoICyE9hObeDA+FVSMuydQB9eBgeAXUOXJUJFX4ae3wS9rAczALnEzcNvSCXvC5sGroRfj3roiHE94d9YRhUW5VxTaz0errgLEq6gsp8CQ6Wc6ELTIO14W1YGlaCERC0MoFvwAHUDpxElBPkF8nRuHIQjMqEcr84yd4MncHnDQcn1jD5/ko46C8CId3F9YdwgqO75FxaisT0dTOSi7w/DGiTTHfxCpoZAhzngM8dArvAgXA3qK4wNhOa9+X7eaPzxGonMb1Nl+134hLq6GUO/k3gMDgRnJROgZugLPqDi4emblglCbfm2ApOTZ3Ts60gtGvq1DzBfGL2Ba8NMq97QoSj/cENi/q58JDZkD1LOHj8C+EXYW14F8qquckNjs0ToDn8BC54em2/KUlbcsFu8CM4UU+EbOmLZdXLIPuNHijrF8pi/AdoZCCRfc7yrAGhTJ8Qfg+cwNN9xnZP90+iGW3L6+FwKNwKl0MfKEo+Mzsf4+m+Hu61vHNChKMeNobNwDadAO+D1wXZ/y3PweA4vhZ6Q1HpnMqpspQzOwPbJMi5Q4/DhmurcIJjP/g6FTfoRs6FuC240fkYghyf6TbRD1VcuufTz7yOeHo8W665Wdc4l3wFLSDdVusRfxCmwnvg8/UpW0uRkL7PPulYcNOg0nUoTPn3a1H+N+BS+3U6/3/fHVOKdcDGWFTkxGOnuB4eTejB8WlwcLkQN4Y6YAd0kjfcA26AdWAUuNApJ1PvOc8IWg/2BHfSRWldTuwOy8NvsAxcBQ784jSGk2uDk67yXZCDPlsuWp3gzeTEZRxHwufgs0eDMmxacXLCXhKOSC5yQHeHvuBEUJRcABomJ/Vz2SSsL/lJ2IOL3FFwEIT8ViB8IPRN2IPjhXATlEVOpgWpGyyTckIZDmFhN620cqK7AMICmOs++0S2nORXzUp04jO9rHJCbpTc1Iujm5GxSbykPpRclln03TScnCQ4eXcLJ1PHSYRt/9YwPkm3j4fn2YfDgmKZ2oH91P6npsNPYBntg+oQ+BGs++pgfWz7tmAbb5/EOWTqaRnPhkthYxgMfSCt0HdM+wwOgJCv49ixYnq2liPBvukYtp6rgePDctwKPlO5MVgG6sPpYHkuh3XBvmSfyJXem/S0sst5JCdDORcjrG+5ypm+L51fOvw5Eetgmf8A+8J5oNdp7ULETV8HsB/bnpZBhbmjMFbYVo7HotJ9pu3uc1R72BXS5bU++qZfv4HP6gs/QLZOI+FB0F/luLd/KfMM5TRPvb8HlHWYDqGu6ed7Ppcs+5pQF5wr6sBacAPYL6Iq4ICmLipy0XoI7LwzwU5yPbQD67k7PAaPwgB4GPaFqXAP9IKREDSYwNXgpOHEcyIcBWoTOAX2NJLSJMJ2/mNgMhwGP0ETKE7nc9IyNIVW0BnegmxdQcL90BCWAMvvpPcrWCcXiLpwEnSH4uQ9A8HnzQLr5KByUH4PPutcyNbrJFwGPs9neK16HJwQboZ3QH/C4v0dYfO7EA6BZvAhWP7noSxy8nSyMY9vwIVhKagPTlZzoAu8DE6uu8IYGAvqT1gebHcnzZag1/pv+WyL36ENbAz3gcoDn+NRjyaA+hgOBes8GtaC1eAOKEnWYxnIAxeWQ+BeULavvqnNoRvcCV+Bsvym65/3ugl8DrwvtIn9qTfoUz2wnYOso+34NNjuHWAzOBkWB699EKx/T7BcUyBIn8+D+8G2bQ7HwqYwHnym9w6Go8Fxp2bAOWDfyod+MAR2AK/Nlte3g7PA59jHLNcgOBBegVGQrdokPAID4ABw7E8E+8KO8CnYVruAbeicsQn0h5dhG7A8RaVzah5ZzjywXheBffFReAr2hzfhPciW9/UG7ytKr3PCMj4J+q1XLaEPpGWdVoYe8BfYL+3vTeAWsAz2jelwJuyRhItLv4xrPoeTYCCkNYLI22Ad7R96tgpcANnS953AcdIR9obJoOznwSP74nD4DTx/GljPuVBavcGFn4Hlsq/sCV/CUOgOURVwoFYF7l0gt/r3m5kzZ5a2nKtTKCeBOvAcvA9LwhHghPE41IX9oDV8CE/DHNgIfoSxELQcgYPADuuk8wUoB+Zt0NVIlixDN/gDHoY2YOd3sG4KT4Jy0UvH1yK+G0wCJ0EHjRNFttYgwTqa/z3wNag1YWdwsnCyHAfKieEV8PkqHd+Q+PbgPcEvgpnFcmuOt8OW8DI4CSr9PASagn62gmHwEzQBJ2jPvQhORmpjML87jCCvaQRjwLL6/KJ0NiechIP0d32wrZ4A6+zzPoAPQZm2AcwG6+4kFNSAQCj/FMI+P9Qtj7B9Y3EYDw/BTFgBtoW0nJCCp/apg8C2/hxsF/tUWn2InJdOIKz39jFlWZ+HGUZQfbCcS8FHMBE6wb0Q5P164bO8Vw+U6WuDZX8ULPtrYJ2ytR0Jm4ATt3l71CMXJvutRxcjvbCdToI1oCeoLaEz/Ar3wxRQS8CBoHdO4vYHZR12BZ+lf/aFZWECmDYLsrUXCSvD1WD/8x7zHQ3Wz3KlZX6OI8u2I3wGD4LX1QPLZX6jwL6bD47VxmDejk3vtx3/hKLSOTWPdifWGq4Bn2NeK4H+mf8cyFYHEvYAvfNe+49eqC7wJdhu5ueC1xYs233wO2RrUxK2Bb31ms1gBEwDy7Iv2F8fh7GgikpfjnP7QSOw/7wKyvwtZwFYrr2hHUyBe8G5y3K2hHCPzzQv6+i4fwnsW5bRvnI4TAbLtSJYTvuh/dryq86gN3qSrSVJ0H+frxYD88gDr9ff2eDz5RWolirjmlgt61ihQi/sL30UUXg73WVFnIvJlevAg2R3BtSq3GwXWG61k/K7aFR3OfFfBVdW8Yq4aXDhiooOLHIOVNE1ser4XEUNqlN1HIoliQ4sMAdu40mfgJ9iVGU1oXCDq3IBY9miA+V1oIquieWtTuXfFw2qfE9jjtGB6EB0IDpQPR1Y2GuiH51GRQeiA9GB6EB0IDoQHSjRgbhpKNGieEF0IDoQHYgORAeiAzqwqG0aelMnv2l8jZWrIvJb9n4bO+hmAn7T+DHwG9yVoaPIpGdlZFSGPNbkWv/GHRUdiA5EB6ID0YGq4UAZ/37zK6XeCPz5TVXRdxQkbA78AqU/81L+BGq1TKjiL1eQxQUVz6ZMOWzJ1e+W6Y54cXQgOhAdiA5UyIEyrokVelaum/0J1aIif5PrZmEdmAR/gL8fdmH+Gu4GNxX+9nh1aA4rwHR4En4CF/dNwN+kqw3gd3Ch3yrBn/4NB39nrLqDv1neBe6BCXAQmNdzkJb5vZNKyCPsb7T9eVh/+B6U91r2huDmwrLPBBV+5/4XYX97XABpNSPib9Mto9c+AWpT8J63jCB/0/wy+Ox9wJ+R6tmjYH3MRy+XgpZwQxLuwVF9UXj412tTUvy3A3yWzxwHltO4Mq0L+CnX2xA8sl0OhPqgR8+CKirdczuBz5oBA8E2VBvD9mB9rP9YiIoORAeiA9GBCjqwqP15Im3HACJHg4u4G4FhsAS0B/9BkQPAxdljV1AHwwPggqmuBBfBQ+FG+Ar85MBrtgN1PXhuaWgALvB7gs/1/kYQtAOB50OE47XghsYF20V+CXAjMxLc0BXA/nATqHx4EX6EP+ENWAmCLKvnfeZ0sCzh+T7L8ih/x34L/AqPgPUvANPfgiXBP6F4/yHQChqCi7ybCDcwV0EuWRf9OQ4+gcMgPHcfwv1gGnwDd0E3qAXW3zq7GbkMjoCi0jmV+WTlHI5fQhsYBvq3IdwDBZDLI5KjogPRgehAdGCRdKCMH8XMwYTG0BZcWH3XGuQCexDkgwu/C5Q6Bu7IhAr/mdHXCO8NbgJc3OqA71rXhKCHCJySRFz0XXSVi6vPddFVLsKzwM2JcmFcJhMqzHvrJOzBRc/nugC66QhyUX0niTzD8dBwguN+0AmuADcu74GLdNBgAv8BNxMfwVRoCG6UBkBtOA5cbJVHy98O3Gj9DIuB6gWPZEKFL26w3k3FQ3B9AjMg5KkHbk7cUFlfzwe5gTgXGoOfeGwDlqklrALFpf/O+eUh6AUC+mEbu7lx86LWBdsyKjoQHYgOVHsHyrgmVnp9naAXRbnofQa+Iw5yQTVdFcBsA+hp2BZcTJeDO8HFbQcYBG5ERsEhMBI+h83Ad8FBpikXuvHgAqi+hcmZ0D+bhe+SuIcPU2HflbvZKYDm8DyMg0sgtJP5e13QfQQ+TiK+M3eBTpfrKeLbgfV5Bd6ELWBn8GP7v+B96A/WcQQsDuF5Ewn/CWpVSD/b+4rSGE64qCs9+AncCFjW7vAWfAFdwPJ6/hToD15/AcyFotL1qR68AfotG4NlfBh8vmX9FHaBdD8gGhUdiA5EB6ID5XEgLA7lubcq3+O75aZZBXRBDQu2G4GgSQRcqI6GYfAyuMiGhZVg5t+tr8NxD3DhHgLpxdnFV5l/+rle08gTaFt4MRP65yV9bTOSfYd+VEJvju3hFAjP+pmwfx4IyiewVhK5meOucCm0StLCpsF38NZLdoAtYDCsAC/AI7A5rA1hk0Bwnn8rP7tuTbygCKXL6OLutdZtYBJ24+DC78bIurlReRZWAr13w3A3FJXuZuIPcJPQOsHwNeCnPCfDMnA87Am9ICo6EB2IDkQHKujAorppeBdfXIzcCCgXxO3gGSM55OJ6Jvhu3E8G3ASkF/nliPsufRK4sO4Ii0G2PiDhFzgsOeHzw8bAxdpFMi03BHXB8m0Jlq8F+M7ZvFwAj4PwrCcJnwQNwU8jboOweLtRej9Ju4Oj9be8bjR2g6HgpuFQ8ON7330vDS7QnvN+F9dGEJ5H8G/5Dn4P6AhLgGUvSl7TLTnp5kfvpoM+joQJ4DW7gs9qCta3Q3IcwdHNS1HpX3DOTxP6gPVcFT6ETuCG5CGoA6+CHrjBUHtBy0wovkQHogPRgejAoudAGf9+46JfP3HBRcl39i5Qb8F2oNaGWzKhf15cbFxgXLDV6XB5JlT44oLvx90ufvfCsXAWqMdh+Uyo8MUFzHIUgIv6g+ACPwpcyIIeI3AuuAC+B11Aea2bCzcAlsmFbhC4uPqu/Sr4Elw0jwB1FPTMhAoX9CcIb5nEj+ToJiLoSQIu1kEXEPgMhoPluRa2Av27E9KyLB+Df47pC9YvW+uTUAAPJUefF/zpTPgj0Ec3IZbtfFAHwGiwvV6C1UAVld6Sc9azALzvIFBuwm4G8xkPN4LeKdti80wovkQHogPRgWroQBnXxGpYwwoWORpUQQMX/O1uGtzQREUHogPRgehAJTuwsNfERfXPE5XcTDG76EB0IDoQHYgORAeqvAMLe1dV5Q2qegX0OxEbVr1ixRJFB6ID0YHq70BcE0tow2hQCQbF09GB6EB0IDpQYxxY2Gti/PNEjelqsaLRgehAdCA6EB2omANx01Ax/+Ld0YHoQHQgOhAdqDEOxE1DjWnqWNHoQHQgOhAdiA5UzIG4aaiYf/Hu6EB0IDoQHYgO1BgH4qahxjR1rGh0IDoQHYgORAcq5kDcNFTMv3h3dCA6EB2IDkQHaowDcdNQY5o6VjQ6EB2IDkQHogMVcyBuGirmX7w7OhAdiA5EB6IDNcaBuGmoMU0dKxodiA5EB6ID0YGKORA3DRXzL94dHYgORAeiA9GBGuNA3DTUmKaOFY0ORAeiA9GB6EDFHIibhor5F++ODkQHogPRgehAjXEgbhpqTFPHikYHogPRgehAdKBiDsRNQ8X8i3dHB6ID0YHoQHSgxjgQNw01pqljRaMD0YHoQHQgOlAxB+KmoWL+xbujA9GB6EB0IDpQYxyIm4aq09RbUJRaVac4sSTRgehAdCA6EB2Y14FFbdOwJtXrAxfNW81KjR1HbkdXao7/93+2wy4wt5Lz7UZ+F1ZSnr3J51roWEn5ZWezEQl3ZSfGeHQgOhAdiA5EB0rtQP369cuzkJ5d6geU70I3JpWp3cls08rMMJVXZZa1C/lunMq7soOVWdbKLlvMLzoQHYgOLHQHyrkmVlq561ZaTtUjoz0o5nowC26Fr0AtAfvBSjAbBsJkCFqbwN7gff1CYnJckeNhsBiMgXsgaC8C7eFC2B/aws0wHdJyw3BqKuFQwp+CC3QjeAGGQ1BrAj2hDgyDQRBkeQ6HevBUSEyODTj6KYnHX+Am+A0qqlCeumT0GqTLswnxzuC59+FpCKpPoBc0hndCYnL0TzUHwqqg77bXNFB5cDCYl+XfF/RnMBSnEzj5MbSATvAr3Ag/gGU5DSbDnWCfMO4zb4eo6EB0IDoQHajqDpRzV3V2jnq5YOyYpLtI3QLNkviSHFsmYRfbS5OwBzcMZ4GLmOcuh8tAef/14MKuNgcXmrT6ErkCNoKG0BHSyieyczqBsGVxM9A8Sbc8LvSqFfhMFzW1P7iAqqXgOnABVMfBo5lQ4Z9AbiS8chJvzdFry6IuXJz9SYNltTx6qFzAD8mECl/acnDDoCynf0IK8vkrJhHzHh1OcDwXNkzi1ut/YLsFed/j4GbLdgnXEixWYzjr90eU/t4OdYygY0Bfgg4ikN1e4Vw8RgeiA9GBBe5AOdfESiunf0uvKXKReS6p7E8czwc/XVC/g59A9AEX/bCQEfy/3eESmAuzwPsWB7U3TAGvcUF0gQwLEsGM3GxcCSPAd/ejIa2diDybTkjCd3GcnoQ/4ui7Y+VC5ibGMqt7YfVMqLA8VxGemcRdaH0XrdYHF8etwbJuCW522kNFZHnOht+STO7n2CEJe7CPnQJ6a7qf5qh14DXQP/UyvJkJFfq7IWGvt6y7wtfJkcPfeo+Q3touI/9OLT7wKqeHJZfo72OwSRLX8x5JWK9Wg+z2Sk7HQ3QgOhAdqHkOhHeANaHm/tkhLRe5sPifRHgUnAcucpdBkIv+3BDh+CfMSeJ/cXwDJiVxD0NTYYNe44KXS2uR6IYgnX+u69LnbbM/si4K5fHTh7B4h0vCBsJyWMd0+QxPDReW81hcedqSZw/Qz5+gCwTPiyur9R0P6bIS/b/vfEmpPGVvmLrfoBunb5M0N2Lm2Qo2gwchKjoQHYgORAcSB1wga4pc7MNHzdb7DHgoqfyyHF9Mwi4YayRhDy5c+6fixxN2UVMPQzfw3XJBQh7HWlAa+VH+A6W5MHXNI4TPTMU3IjwjifuuuReE5+9I2IVbvQfW/0coSHABbQwVkeXRyyA/IfguifgJyFPghsHNxfYQNJJAV1gySWiXxI3aVtbJjUVBghulFaG0Oo4L9cPnptWByOGwDPjnpE3hbQjqT6An2Ac+gqjoQHQgOhAdSBwIi0uVNcS/38ycObO05fR7C/79W/xY3nfXl4CfMtQBF5KmUA98F/kBqE7QHbzuG3DRUncUHjJ/fvDjdMvhnxJ2hsHwGrQBF5lZ4JchTfOccnFvAZZFDQIXS9UadgL/hJCt3iS4qF0JLsAXwCTwOwnKhc4y+AnDZLgNwkZmA8JuZKyLnyw0B/O4H5rA8VAbXEy/hAGgT8XJ520LLvBe6wKerku6PG6gbgXLo+engou/8lOZbeA6+ArcBBwDlvVbmApu2q4C7/WcGzrLax1uh1/APLYCy/Mz2GY3QVreuzXsA+Yf1IfAU2B9rP8TkD5P9P/6wtPwLkRFB6ID0YEq40AZ18QqU+4FVpCF/aWP+VjRzci7wXzMP2ad2wE3DcXJjeFFxV0Qz0UHogPRgYXlwMJeE30HF7VwHBjOY39dOI+usU89kZqvDWeDm4O0/DTEDcXV0BJWg6joQHQgOhAdSDmQPXGmTlWNYPwoZoG0w8E8Ja+IJ/lnjc+KOBeTowPRgehAdGABOhDXxBLMXtgfxZRQvHg6OhAdiA5EB6IDC8yBhb0mxj9PLLCmjg+KDkQHogPRgehA9XYgbhqqTvvVrzpFiSWJDkQHogPRgejAvx1Y1DYNramif5/f5d9VrbQU/3XCjSott38yOpdg+jsm+xG3Lv7kMJf8Qt/WuU4Uk1aRsu9LvpanWTH5V+RU+yT/iuQR740ORAeiA9GB+ejAorZpmI5XQ2Hj+ejZO+Ttv69QmVqFzMbD3FSmrxP230Rok0qraLAiZffXHv57Bm0rWogi7vffbXAjFBUdiA5EB6IDVdSBRW3T4E8YCyD808nZtvuzOv8holwLn/8w0/rgAp5La5Hoz/CmQPhnm8N15rc5ZP+JwX9MqVVykedCOEn6+3AQoYF/xwoDEzlMy0oL0aUJ/ABvh4TkWI/j8rA4+O9A+A9LpZUuu59g5EH60430tdnh4srjtXrjZk0fs5VHgp/O6H+2/OTCNvEfjLJOafnvWGwBrdOJhC1zHjQE1Qasc0nyWXpnW5jvcpDWSkT8R7WCViTgPwYWFR2IDkQHogM4ULsGueC72CvARWBHuBDCgulCciq4yG4AJ0GQC9f/wEVxE7gY/NcKlfdfAObnBsE8XayDzMvf/XeFvrATbAlpuXD9CL+lE0sI+4wdoEfWdW5eboVLwYXxZFgPcqk/ifuAC2NFpBf+C45bgXW5HtIbrwOJ50Nj0Lv0ItyD+GHgQn0RdIKgrQn0ATcG/lnFP98EubjvBseCz+sMvaAkmefdcAFYnoMhtLV+2c5dQLnBOQGONBIVHYgORAeiA9XAgXL+vOTsHFW7gbSwSfD0NtDNQEp5hH036mIVdAqB9DtSNx/mpVzMXLDScmOS1vlE9k0nZIUt69JZaSG6LQH/L4dcctHLtVBeS3rd5Abre04SDgcXzL7gu/uyygV146ybDiG+fiqtHmHLkJYbhTzwmW52lBs0N2pBlnVYEjF8TTiRHN2Y5afS3Ow8C+HThtSpYoODs87qYZskzc2Om5ggNzJ1QiQeowPRgejAwnagnGtipRU7LC6VlmEVzuhnyjY3VT4XqDPgcfBPC91hFKh1Cg+Z1ya8TkvFPyAcPhVYlbALYvrTAxeZ2vAXqDlwfyb07xcXfq///t+nyp1iXrOTu61vKEfIcDMC70NBSKjgsR3390vlMYtw+POQ79YvganwNbi5GASqI7ybCRW+WNa3krjv+teEPkncg57aZmk9TuSXdEIpwp9kXfMxcTcNX8Ln4Ccjbnxs27Fg+0VFB6ID0YHoAA7UpE2D70jTi/k2xMOitT3h9Lv2NYgHfUfAj8MnJwm+8/c/SlIuYi4uA42UQ4dyz13luK8itwzlZv9MIFdBekNEtMwazR1+gvBGcufiHN0sqB3gbnjPCHLjYBsoF2vr/6oRZPoWmVDhP69tvhdA9qYnuaTEQ3uucEPzTNaVfmJxKUwHn/kfSG9ObEs3Dm2z0olGRQeiA9GBmu3AorZpOJHmbAJbgnXz3auLo++8b4Ib4HVwE+C7Sc+pCXAM/ADrgh9THw63wy3gIvMh+P0G83dB2QpcgNeCs8B3qua7LPQGn30m5EPQ8wRGJBEXV/8UMjmJpw+bE+kCPud36Aq+Qx8JeXAweP/a4PcBLNsT4D354GbIxXI32BoGgM/xOwDWwU8DHgQ/3v8f9IPitBknt4E28Ce4yQp1uZ9wX9gIvgE3EPql3oGTwHq0A++vD5Z3Cuj7uTAO1oMv4FS4Em6Ea8HNiBu+1eBh0APLoie204rwLXh9Wv8lshfYXnoY9BQB21r/9OIu+AmCLIttPxZmh8R4jA5EB6ID0YFq4EAl//2mFlVuDS602WpBQqvsxFR8JcLNUvF0sBERF0QXqNJqby7sUNqLq8F1bpb0NltuEkx3gc6lxiR6vk6uk8k5vS+rfG6u9upTioz6ck29UlwXL4kORAeiAwvUgUpeExdo2RfIw6JBC8TmmvIQPw3xU5XNc1TYTctW4J8r/FQq18aS5KjoQHQgOrDwHFjYa2LdhVf1+OQq5EBHytK8iPL4p46fizhX3ZJrU+AB4AahFsyFIM+Z5p8qHoKiPvngVFR0IDoQHaiZDsRNQ81s9+xaL5GdkIovSXhR2TS8mapXdtDveQzJTozx6EB0IDoQHfjHgbhp+MeLmhwKv26oyR7EukcHogPRgehACQ74kWxUdCA6EB2IDkQHogPRgRIdiJuGEi2KF0QHogPRgehAdCA6oANx0xD7QXQgOhAdiA5EB6IDpXKgOnynYcbC/olJqZyMF0UHogPRgehAdGD+OzBj/j8iPiE6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdKAYB/4ffeRvbLZiFvMAAAAASUVORK5CYII=", + "text/plain": [ + "" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Image(filename='../../../docs/source/_figures/remote_3.png')" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.loader import RAGQueryLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "query_loader = RAGQueryLoader(\n", + " data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store\n", + " # Arguments to pass into the seed node/edge retrieval methods for the FeatureStore.\n", + " # In this case, it's k for the KNN on the nodes and edges.\n", + " seed_nodes_kwargs={\"k_nodes\": 10}, seed_edges_kwargs={\"k_edges\": 10}, \n", + " # Arguments to pass into the GraphStore's Neighbor sampling method.\n", + " # In this case, the GraphStore implements a NeighborLoader, so it takes the same arguments.\n", + " sampler_kwargs={\"num_neighbors\": [40]*3},\n", + " # Arguments to pass into the FeatureStore's feature loading method.\n", + " loader_kwargs={},\n", + " # An optional local transform that can be applied on the returned subgraph.\n", + " local_filter=None,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make better sense of this loader's arguments, let's take a closer look at the retrieval process for a remote backend:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "SVG(filename=\"media/remote_backend.svg\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we see here, there are 3 important steps to any remote backend procedure for graphs:\n", + "1. Retrieve the seed nodes and edges to begin our retrieval process from.\n", + "2. Traverse the graph neighborhood of the seed nodes/edges to gather local context.\n", + "3. Fetch the features associated with the subgraphs obtained from the traversal.\n", + "\n", + "We can see that our Query Loader construction allows us to specify unique hyperparameters for each unique step in this retrieval." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can submit our queries to the remote backend to retrieve our subgraphs:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/10 [00:00 \ No newline at end of file From 4faad32b605ef7829da7f645630392581ff613d9 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 02:25:52 -0700 Subject: [PATCH 662/752] rag generate running on whole dataset now --- examples/llm_plus_gnn/rag_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index d4bb628dec56..8d0148047d6b 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -14,7 +14,7 @@ import pandas as pd # %% -ds = UpdatedWebQSPDataset("small_dataset", force_reload=True, limit=20) +ds = UpdatedWebQSPDataset("full_dataset") # %% triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) From 567b43eeb56079424bfc97141e3a3c3bc6187c03 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 03:52:41 -0700 Subject: [PATCH 663/752] docstrings --- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 318 ++++++++++++++++-- .../llm_plus_gnn/doc/media/remote_backend.svg | 2 +- examples/llm_plus_gnn/profiling_utils.py | 33 +- examples/llm_plus_gnn/rag_feature_store.py | 36 +- examples/llm_plus_gnn/rag_generate.py | 57 +++- .../llm_plus_gnn/rag_generate_multihop.py | 34 +- examples/llm_plus_gnn/rag_graph_store.py | 17 +- torch_geometric/loader/rag_loader.py | 18 + 8 files changed, 454 insertions(+), 61 deletions(-) diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb index 72912715646d..fc3451fa89e4 100644 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -49,7 +49,7 @@ "" ] }, - "execution_count": 26, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -86,9 +86,50 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Saving the dataset (1/1 shards): 100%|██████████| 10/10 [00:00<00:00, 923.25 examples/s] \n", + "Processing...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Encoding graph...\n", + "Saving graph...\n", + "Encoding questions...\n", + "Retrieving subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 31.85it/s]00:00" ] }, - "execution_count": 23, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -19873,7 +19914,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -19882,7 +19923,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -19910,7 +19951,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 50, "metadata": {}, "outputs": [ { @@ -19922,7 +19963,7 @@ "" ] }, - "execution_count": 29, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -19952,7 +19993,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -19961,7 +20002,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 52, "metadata": {}, "outputs": [ { @@ -19983,7 +20024,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 10/10 [00:06<00:00, 1.49it/s]\n" + "100%|██████████| 10/10 [00:07<00:00, 1.28it/s]\n" ] } ], @@ -19995,16 +20036,16 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Data(x=[2193, 1024], edge_index=[2, 7295], edge_attr=[7295, 1024], node_idx=[2193], edge_idx=[7295])" + "Data(x=[2251, 1024], edge_index=[2, 7806], edge_attr=[7806, 1024], node_idx=[2251], edge_idx=[7806])" ] }, - "execution_count": 32, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -20017,7 +20058,230 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "These subgraphs are now retrieved using a different retrieval method when compared to the original WebQSP dataset. Can we compare the properties of this method to the original WebQSPDataset's retrieval method?" + "These subgraphs are now retrieved using a different retrieval method when compared to the original WebQSP dataset. Can we compare the properties of this method to the original WebQSPDataset's retrieval method? Let's compare some basics properties of the subgraphs:" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import Data" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "def _eidx_helper(subg: Data, ground_truth: Data):\n", + " subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx\n", + " if isinstance(subg_eidx, torch.Tensor):\n", + " subg_eidx = subg_eidx.tolist()\n", + " if isinstance(gt_eidx, torch.Tensor):\n", + " gt_eidx = gt_eidx.tolist()\n", + " subg_e = set(subg_eidx)\n", + " gt_e = set(gt_eidx)\n", + " return subg_e, gt_e\n", + "def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int):\n", + " subg_e, gt_e = _eidx_helper(subg, ground_truth)\n", + " total_e = set(range(num_edges))\n", + " tp = len(subg_e & gt_e)\n", + " tn = len(total_e-(subg_e | gt_e))\n", + " return (tp+tn)/num_edges\n", + "def check_retrieval_precision(subg: Data, ground_truth: Data):\n", + " subg_e, gt_e = _eidx_helper(subg, ground_truth)\n", + " return len(subg_e & gt_e) / len(subg_e)\n", + "def check_retrieval_recall(subg: Data, ground_truth: Data):\n", + " subg_e, gt_e = _eidx_helper(subg, ground_truth)\n", + " return len(subg_e & gt_e) / len(gt_e)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "from torch_geometric.data import get_features_for_triplets_groups" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet)\n", + "num_edges = len(ds.indexer._edges)" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 60.20it/s]\n", + "1it [00:00, 1.18it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2193, Ground Truth Size: 1709, Accuracy: 0.6636780705203827, Precision: 0.22923807012918535, Recall: 0.1994037381034285\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2it [00:01, 1.41it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2682, Ground Truth Size: 1251, Accuracy: 0.7158736400576746, Precision: 0.10843513670738801, Recall: 0.22692963233503774\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "3it [00:02, 1.51it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2087, Ground Truth Size: 1285, Accuracy: 0.7979813868134749, Precision: 0.0547879177377892, Recall: 0.15757855822550831\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "4it [00:02, 1.56it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2975, Ground Truth Size: 1988, Accuracy: 0.6956088609254162, Precision: 0.14820555621795636, Recall: 0.21768826619964973\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "5it [00:03, 1.59it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2594, Ground Truth Size: 633, Accuracy: 0.78849128326124, Precision: 0.04202616198163095, Recall: 0.2032301480484522\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "6it [00:03, 1.61it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2462, Ground Truth Size: 1044, Accuracy: 0.7703499803381832, Precision: 0.07646643109540636, Recall: 0.19551861221539574\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "7it [00:04, 1.62it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2011, Ground Truth Size: 1382, Accuracy: 0.7871804954777821, Precision: 0.10117783355860205, Recall: 0.13142713819914723\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "8it [00:05, 1.63it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2011, Ground Truth Size: 1052, Accuracy: 0.802831301612269, Precision: 0.06452691407556001, Recall: 0.16702726092600606\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "9it [00:05, 1.64it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 2892, Ground Truth Size: 1012, Accuracy: 0.7276182985974571, Precision: 0.10108615156751419, Recall: 0.20860927152317882\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:06, 1.58it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 1817, Ground Truth Size: 1978, Accuracy: 0.7530475815965395, Precision: 0.1677807486631016, Recall: 0.11696178937558248\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "for subg, ground_truth in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)):\n", + " print(f\"Size: {len(subg.x)}, Ground Truth Size: {len(ground_truth.x)}, Accuracy: {check_retrieval_accuracy(subg, ground_truth, num_edges)}, Precision: {check_retrieval_precision(subg, ground_truth)}, Recall: {check_retrieval_recall(subg, ground_truth)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that, since we're only comparing the results of 10 graphs here, this retrieval algorithm is not taking into account the full corpus of nodes in the dataset. If you want to see a full example, look at `rag_generate.py`, or `rag_generate_multihop.py` These examples generate datasets for the entirety of the WebQSP dataset, or the WikiData Multihop datasets that are discussed in Section 0." ] }, { diff --git a/examples/llm_plus_gnn/doc/media/remote_backend.svg b/examples/llm_plus_gnn/doc/media/remote_backend.svg index d8d4ad8a3b37..c5791f0a95de 100644 --- a/examples/llm_plus_gnn/doc/media/remote_backend.svg +++ b/examples/llm_plus_gnn/doc/media/remote_backend.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/profiling_utils.py index 2bc537f08d4a..0f1c0e1b87ec 100644 --- a/examples/llm_plus_gnn/profiling_utils.py +++ b/examples/llm_plus_gnn/profiling_utils.py @@ -100,6 +100,7 @@ class RemoteDataType(Enum): @dataclass class RemoteGraphBackendLoader: + """Utility class to load triplets into a RAG Backend.""" path: str datatype: RemoteDataType graph_store_type: Type[ConvertableGraphStore] @@ -140,7 +141,37 @@ def create_remote_backend_from_triplets( node_method_kwargs: Optional[Dict[str, Any]] = None, edge_method_kwargs: Optional[Dict[str, Any]] = None ) -> RemoteGraphBackendLoader: - + """Utility function that can be used to create a RAG Backend from triplets. + + Args: + triplets (Iterable[TripletLike]): Triplets to load into the RAG + Backend. + node_embedding_model (Module): Model to embed nodes into a feature + space. + edge_embedding_model (Module | None, optional): Model to embed edges + into a feature space. Defaults to the node model. + graph_db (Type[ConvertableGraphStore], optional): GraphStore class to + use. Defaults to LocalGraphStore. + feature_db (Type[ConvertableFeatureStore], optional): FeatureStore + class to use. Defaults to LocalFeatureStore. + node_method_to_call (str, optional): method to call for embeddings on + the node model. Defaults to "forward". + edge_method_to_call (str | None, optional): method to call for + embeddings on the edge model. Defaults to the node method. + pre_transform (Callable[[TripletLike], TripletLike] | None, optional): + optional preprocessing function for triplets. Defaults to None. + path (str, optional): path to save resulting stores. Defaults to ''. + n_parts (int, optional): Number of partitons to store in. + Defaults to 1. + node_method_kwargs (Optional[Dict[str, Any]], optional): args to pass + into node encoding method. Defaults to None. + edge_method_kwargs (Optional[Dict[str, Any]], optional): args to pass + into edge encoding method. Defaults to None. + + Returns: + RemoteGraphBackendLoader: Loader to load RAG backend from disk or + memory. + """ # Will return attribute errors for missing attributes if not issubclass(graph_db, ConvertableGraphStore): getattr(graph_db, "from_data") diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index ca3a0ba646d4..74eb73ec99eb 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -35,6 +35,17 @@ def edge_attr(self) -> Tensor: return self.get_tensor(group_name=(None, None), attr_name='edge_attr') def retrieve_seed_nodes(self, query: Any, k_nodes: int = 5) -> InputNodes: + """Retrieve the seed nodes for a given query using KNN search after + embedding the query. + + Args: + query (Any): Input to be embedded and used for KNN search. + k_nodes (int, optional): Number of nodes to search for in KNN + search. Defaults to 5. + + Returns: + InputNodes: Input object to be used in a PyG Sampler. + """ return next(self._retrieve_seed_nodes_batch([query], k_nodes)) def _retrieve_seed_nodes_batch(self, query: Iterable[Any], @@ -51,6 +62,17 @@ def _retrieve_seed_nodes_batch(self, query: Iterable[Any], yield indices def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: + """Retrieve the seed edges for a given query using KNN search after + embedding the query. + + Args: + query (Any): Input to be embedded and used for KNN search. + k_edges (int, optional): Number of edges to search for in KNN + search. Defaults to 3. + + Returns: + InputNodes: Input object to be used in a PyG Sampler. + """ return next(self._retrieve_seed_edges_batch([query], k_edges)) def _retrieve_seed_edges_batch(self, query: Iterable[Any], @@ -71,7 +93,17 @@ def _retrieve_seed_edges_batch(self, query: Iterable[Any], def load_subgraph( self, sample: Union[SamplerOutput, HeteroSamplerOutput] ) -> Union[Data, HeteroData]: + """Retrieve subgraph features corresponding to the given sampler + output. + + Args: + sample (Union[SamplerOutput, HeteroSamplerOutput]): Output from a + PyG Sampler to retrieve subgraph features for. + Returns: + Union[Data, HeteroData]: Data object containing subgraph features + and edge indices. + """ if isinstance(sample, HeteroSamplerOutput): raise NotImplementedError @@ -89,5 +121,7 @@ def load_subgraph( class SentenceTransformerFeatureStore(KNNRAGFeatureStore): def __init__(self, *args, **kwargs): - kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') + kwargs['model_name'] = \ + kwargs.get('model_name', + 'sentence-transformers/all-roberta-large-v1') super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 8d0148047d6b..10412dfea87a 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -1,41 +1,54 @@ # %% +from itertools import chain + +import pandas as pd +import torch +import tqdm from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import RAGQueryLoader + +from torch_geometric.data import Data, get_features_for_triplets_groups from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet +from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd # %% ds = UpdatedWebQSPDataset("full_dataset") # %% -triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) +triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) # %% questions = ds.raw_dataset['question'] questions # %% -ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +ground_truth_graphs = get_features_for_triplets_groups( + ds.indexer, (d['graph'] for d in ds.raw_dataset), + pre_transform=preprocess_triplet) num_edges = len(ds.indexer._edges) # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) +model = SentenceTransformer( + model_name='sentence-transformers/all-roberta-large-v1').to(device) # %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() +fs, gs = create_remote_backend_from_triplets( + triplets=triplets, node_embedding_model=model, + node_method_to_call="encode", path="backend", + pre_transform=preprocess_triplet, node_method_kwargs={ + "batch_size": 256 + }, graph_db=NeighborSamplingRAGGraphStore, + feature_db=SentenceTransformerFeatureStore).load() # %% -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}) +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, + seed_edges_kwargs={"k_edges": 10}, + sampler_kwargs={"num_neighbors": [40] * 3}) + # %% # Accuracy Metrics to be added to Profiler @@ -48,28 +61,36 @@ def _eidx_helper(subg: Data, ground_truth: Data): subg_e = set(subg_eidx) gt_e = set(gt_eidx) return subg_e, gt_e + + def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): subg_e, gt_e = _eidx_helper(subg, ground_truth) total_e = set(range(num_edges)) tp = len(subg_e & gt_e) - tn = len(total_e-(subg_e | gt_e)) - return (tp+tn)/num_edges + tn = len(total_e - (subg_e | gt_e)) + return (tp + tn) / num_edges + + def check_retrieval_precision(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(subg_e) + + def check_retrieval_recall(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) + # %% retrieval_stats = {"precision": [], "recall": [], "accuracy": []} subgs = [] -for subg, gt in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): +for subg, gt in tqdm.tqdm( + zip((query_loader.query(q) for q in questions), ground_truth_graphs)): retrieval_stats["precision"].append(check_retrieval_precision(subg, gt)) retrieval_stats["recall"].append(check_retrieval_recall(subg, gt)) - retrieval_stats["accuracy"].append(check_retrieval_accuracy(subg, gt, num_edges)) + retrieval_stats["accuracy"].append( + check_retrieval_accuracy(subg, gt, num_edges)) subgs.append(subg) pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') torch.save(subgs, 'subg_results.pt') - diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/rag_generate_multihop.py index 3640dca39166..8dfabc52647a 100644 --- a/examples/llm_plus_gnn/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/rag_generate_multihop.py @@ -1,17 +1,17 @@ # %% +import pandas as pd +import torch +import tqdm from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore + +from torch_geometric.datasets.updated_web_qsp_dataset import ( + preprocess_triplet, + retrieval_via_pcst, +) from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd - # %% triplets = torch.load('wikimultihopqa_full_graph.pt') @@ -22,17 +22,27 @@ # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) +model = SentenceTransformer( + model_name='sentence-transformers/all-roberta-large-v1').to(device) # %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() +fs, gs = create_remote_backend_from_triplets( + triplets=triplets, node_embedding_model=model, + node_method_to_call="encode", path="backend", + pre_transform=preprocess_triplet, node_method_kwargs={ + "batch_size": 256 + }, graph_db=NeighborSamplingRAGGraphStore, + feature_db=SentenceTransformerFeatureStore).load() # %% -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=retrieval_via_pcst) +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, + seed_edges_kwargs={"k_edges": 10}, + sampler_kwargs={"num_neighbors": [40] * 3}, + local_filter=retrieval_via_pcst) # %% subgs = [] -for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): +for subg in tqdm.tqdm(query_loader.query(q) for q in questions): subgs.append(subg) torch.save(subgs, 'subg_results.pt') diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/rag_graph_store.py index ee780aff0860..48473f287233 100644 --- a/examples/llm_plus_gnn/rag_graph_store.py +++ b/examples/llm_plus_gnn/rag_graph_store.py @@ -68,6 +68,20 @@ def sample_subgraph( self, seed_nodes: InputNodes, seed_edges: InputEdges, num_neighbors: Optional[NumNeighborsType] = None ) -> Union[SamplerOutput, HeteroSamplerOutput]: + """Sample the graph starting from the given nodes and edges using the + in-built NeighborSampler. + + Args: + seed_nodes (InputNodes): Seed nodes to start sampling from. + seed_edges (InputEdges): Seed edges to start sampling from. + num_neighbors (Optional[NumNeighborsType], optional): Parameters + to determine how many hops and number of neighbors per hop. + Defaults to None. + + Returns: + Union[SamplerOutput, HeteroSamplerOutput]: NeighborSamplerOutput + for the input. + """ if not self._sampler_is_initialized: self._init_sampler() if num_neighbors is not None: @@ -81,8 +95,9 @@ def sample_subgraph( device = seed_nodes.device # TODO: Call sample_from_edges for seed_edges + # Turning them into nodes for now. seed_edges = self.edge_index.to(device).T[seed_edges.to( - device)].reshape((-1)) + device)].reshape(-1) seed_nodes = torch.cat((seed_nodes, seed_edges), dim=0) seed_nodes = seed_nodes.unique().contiguous() diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index f4e87f83404b..33d6cf0e868e 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -58,6 +58,24 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], seed_edges_kwargs: Optional[Dict[str, Any]] = None, sampler_kwargs: Optional[Dict[str, Any]] = None, loader_kwargs: Optional[Dict[str, Any]] = None): + """Loader meant for making queries from a remote backend. + + Args: + data (Tuple[RAGFeatureStore, RAGGraphStore]): Remote FeatureStore + and GraphStore to load from. Assumed to conform to the + protocols listed above. + local_filter (Optional[Callable[[Data, Any], Data]], optional): + Optional local transform to apply to data after retrieval. + Defaults to None. + seed_nodes_kwargs (Optional[Dict[str, Any]], optional): Paramaters + to pass into process for fetching seed nodes. Defaults to None. + seed_edges_kwargs (Optional[Dict[str, Any]], optional): Parameters + to pass into process for fetching seed edges. Defaults to None. + sampler_kwargs (Optional[Dict[str, Any]], optional): Parameters to + pass into process for sampling graph. Defaults to None. + loader_kwargs (Optional[Dict[str, Any]], optional): Parameters to + pass into process for loading graph features. Defaults to None. + """ fstore, gstore = data self.feature_store = fstore self.graph_store = gstore From 0bdcc6db0233000dd3280950a42a0103eb09ca8a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 23 Aug 2024 16:33:52 -0700 Subject: [PATCH 664/752] drop performance ref --- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb index fc3451fa89e4..d0f6cc3c2e4d 100644 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -27,7 +27,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph, and conduct some experiments to evaluate its runtime performance and its effects on hallucination rate." + "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph." ] }, { From 36426bdbabd859ede462cb55cf345a7a56875c6a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:42:09 -0700 Subject: [PATCH 665/752] Add back in nvtx profiler code --- torch_geometric/profile/__init__.py | 2 ++ torch_geometric/profile/nvtx.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 torch_geometric/profile/nvtx.py diff --git a/torch_geometric/profile/__init__.py b/torch_geometric/profile/__init__.py index 833ee657d0e7..22d3039f4c83 100644 --- a/torch_geometric/profile/__init__.py +++ b/torch_geometric/profile/__init__.py @@ -20,6 +20,7 @@ get_gpu_memory_from_nvidia_smi, get_model_size, ) +from .nvtx import nvtxit __all__ = [ 'profileit', @@ -38,6 +39,7 @@ 'get_gpu_memory_from_nvidia_smi', 'get_gpu_memory_from_ipex', 'benchmark', + 'nvtxit', ] classes = __all__ diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py new file mode 100644 index 000000000000..122ddfad916f --- /dev/null +++ b/torch_geometric/profile/nvtx.py @@ -0,0 +1,55 @@ +from functools import wraps +from typing import Optional + +import torch + +CUDA_PROFILE_STARTED = False + + +def begin_cuda_profile(): + global CUDA_PROFILE_STARTED + prev_state = CUDA_PROFILE_STARTED + if prev_state is False: + CUDA_PROFILE_STARTED = True + torch.cuda.cudart().cudaProfilerStart() + return prev_state + + +def end_cuda_profile(prev_state: bool): + global CUDA_PROFILE_STARTED + CUDA_PROFILE_STARTED = prev_state + if prev_state is False: + torch.cuda.cudart().cudaProfilerStop() + + +def nvtxit(name: Optional[str] = None, n_warmups: int = 0, + n_iters: Optional[int] = None): + def nvtx(func): + + nonlocal name + iters_so_far = 0 + if name is None: + name = func.__name__ + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal iters_so_far + if not torch.cuda.is_available(): + return func(*args, **kwargs) + elif iters_so_far < n_warmups: + iters_so_far += 1 + return func(*args, **kwargs) + elif n_iters is None or iters_so_far < n_iters + n_warmups: + prev_state = begin_cuda_profile() + torch.cuda.nvtx.range_push(f"{name}_{iters_so_far}") + result = func(*args, **kwargs) + torch.cuda.nvtx.range_pop() + end_cuda_profile(prev_state) + iters_so_far += 1 + return result + else: + return func(*args, **kwargs) + + return wrapper + + return nvtx From 0927f480b10a92f704533599631e3c578ba9e039 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:46:34 -0700 Subject: [PATCH 666/752] forgot a file --- examples/llm_plus_gnn/nvtx_run.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 examples/llm_plus_gnn/nvtx_run.sh diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_run.sh new file mode 100755 index 000000000000..b40c32abb0ea --- /dev/null +++ b/examples/llm_plus_gnn/nvtx_run.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Check if the user provided a Python file +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Check if the provided file exists +if [[ ! -f "$1" ]]; then + echo "Error: File '$1' does not exist." + exit 1 +fi + +# Check if the provided file is a Python file +if [[ ! "$1" == *.py ]]; then + echo "Error: '$1' is not a Python file." + exit 1 +fi + +# Get the base name of the Python file +python_file=$(basename "$1") + +# Run nsys profile on the Python file +nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cuda-memory-usage true --cudabacktrace all --force-overwrite true --output=profile_${python_file%.py} python "$1" + +echo "Profile data saved as profile_${python_file%.py}.nsys-rep" \ No newline at end of file From 109420bbb3ed8172305db0bc8e1e742f6d2a85e4 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:51:09 -0700 Subject: [PATCH 667/752] forgot a few more test scripts --- .../llm_plus_gnn/test_nvtx_rag_backend.py | 101 ++++++++++++++++++ examples/llm_plus_gnn/test_nvtx_uwebqsp.py | 27 +++++ examples/llm_plus_gnn/test_nvtx_webqsp.py | 19 ++++ 3 files changed, 147 insertions(+) create mode 100644 examples/llm_plus_gnn/test_nvtx_rag_backend.py create mode 100644 examples/llm_plus_gnn/test_nvtx_uwebqsp.py create mode 100644 examples/llm_plus_gnn/test_nvtx_webqsp.py diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/test_nvtx_rag_backend.py new file mode 100644 index 000000000000..b0f89bfcca61 --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_rag_backend.py @@ -0,0 +1,101 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import rag_loader +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +from torch_geometric.profile.nvtx import nvtxit +import torch +import argparse +from typing import Tuple + +# %% +# Patch FeatureStore and GraphStore + +SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_nodes) +SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_edges) +SentenceTransformerFeatureStore.load_subgraph = nvtxit()(SentenceTransformerFeatureStore.load_subgraph) +NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()(NeighborSamplingRAGGraphStore.sample_subgraph) +rag_loader.RAGQueryLoader.query = nvtxit()(rag_loader.RAGQueryLoader.query) + +# %% +ds = UpdatedWebQSPDataset("small_ds_1", force_reload=True, limit=10) + +# %% +triplets = list(chain.from_iterable((d['graph'] for d in ds.raw_dataset))) + +# %% +questions = ds.raw_dataset['question'] + +# %% +ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +num_edges = len(ds.indexer._edges) + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer().to(device) + +# %% +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +from torch_geometric.datasets.updated_web_qsp_dataset import retrieval_via_pcst + +@nvtxit() +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + q_emb = model.encode(query) + textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph["desc"] = desc + return graph + +# %% +query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}, local_filter=apply_retrieval_via_pcst) + +# %% +# Accuracy Metrics to be added to Profiler +def _eidx_helper(subg: Data, ground_truth: Data): + subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx + if isinstance(subg_eidx, torch.Tensor): + subg_eidx = subg_eidx.tolist() + if isinstance(gt_eidx, torch.Tensor): + gt_eidx = gt_eidx.tolist() + subg_e = set(subg_eidx) + gt_e = set(gt_eidx) + return subg_e, gt_e +def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + total_e = set(range(num_edges)) + tp = len(subg_e & gt_e) + tn = len(total_e-(subg_e | gt_e)) + return (tp+tn)/num_edges +def check_retrieval_precision(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(subg_e) +def check_retrieval_recall(subg: Data, ground_truth: Data): + subg_e, gt_e = _eidx_helper(subg, ground_truth) + return len(subg_e & gt_e) / len(gt_e) + + +# %% + +@nvtxit() +def _run_eval(): + for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): + print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + _run_eval() + else: + _run_eval() \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py new file mode 100644 index 000000000000..6204fe603b84 --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_uwebqsp.py @@ -0,0 +1,27 @@ +from torch_geometric.datasets import updated_web_qsp_dataset +from torch_geometric.profile import nvtxit +import torch +import argparse + +# Apply Patches +updated_web_qsp_dataset.UpdatedWebQSPDataset.process = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset.process) +updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._build_graph) +updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs = nvtxit()(updated_web_qsp_dataset.UpdatedWebQSPDataset._retrieve_subgraphs) +updated_web_qsp_dataset.SentenceTransformer.encode = nvtxit()(updated_web_qsp_dataset.SentenceTransformer.encode) +updated_web_qsp_dataset.retrieval_via_pcst = nvtxit()(updated_web_qsp_dataset.retrieval_via_pcst) + +updated_web_qsp_dataset.get_features_for_triplets_groups = nvtxit()(updated_web_qsp_dataset.get_features_for_triplets_groups) +updated_web_qsp_dataset.LargeGraphIndexer.get_node_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features) +updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features) +updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_edge_features_iter) +updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter = nvtxit()(updated_web_qsp_dataset.LargeGraphIndexer.get_node_features_iter) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) + else: + ds = updated_web_qsp_dataset.UpdatedWebQSPDataset('update_ds', force_reload=True) \ No newline at end of file diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/test_nvtx_webqsp.py new file mode 100644 index 000000000000..1db8e5934728 --- /dev/null +++ b/examples/llm_plus_gnn/test_nvtx_webqsp.py @@ -0,0 +1,19 @@ +from torch_geometric.datasets import web_qsp_dataset +from torch_geometric.profile import nvtxit +import torch +import argparse + +# Apply Patches +web_qsp_dataset.retrieval_via_pcst = nvtxit()(web_qsp_dataset.retrieval_via_pcst) +web_qsp_dataset.WebQSPDataset.process = nvtxit()(web_qsp_dataset.WebQSPDataset.process) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--capture-torch-kernels", "-k", action="store_true") + args = parser.parse_args() + if args.capture_torch_kernels: + with torch.autograd.profiler.emit_nvtx(): + ds = web_qsp_dataset.WebQSPDataset('baseline') + else: + ds = web_qsp_dataset.WebQSPDataset('baseline') \ No newline at end of file From d7ea7ef66bc3269767734c0ec119a19fb2116406 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 19:42:11 -0700 Subject: [PATCH 668/752] rename to not get detected by pytest --- .../{test_nvtx_rag_backend.py => nvtx_rag_backend_example.py} | 0 .../{test_nvtx_uwebqsp.py => nvtx_uwebqsp_example.py} | 0 .../llm_plus_gnn/{test_nvtx_webqsp.py => nvtx_webqsp_example.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename examples/llm_plus_gnn/{test_nvtx_rag_backend.py => nvtx_rag_backend_example.py} (100%) rename examples/llm_plus_gnn/{test_nvtx_uwebqsp.py => nvtx_uwebqsp_example.py} (100%) rename examples/llm_plus_gnn/{test_nvtx_webqsp.py => nvtx_webqsp_example.py} (100%) diff --git a/examples/llm_plus_gnn/test_nvtx_rag_backend.py b/examples/llm_plus_gnn/nvtx_rag_backend_example.py similarity index 100% rename from examples/llm_plus_gnn/test_nvtx_rag_backend.py rename to examples/llm_plus_gnn/nvtx_rag_backend_example.py diff --git a/examples/llm_plus_gnn/test_nvtx_uwebqsp.py b/examples/llm_plus_gnn/nvtx_uwebqsp_example.py similarity index 100% rename from examples/llm_plus_gnn/test_nvtx_uwebqsp.py rename to examples/llm_plus_gnn/nvtx_uwebqsp_example.py diff --git a/examples/llm_plus_gnn/test_nvtx_webqsp.py b/examples/llm_plus_gnn/nvtx_webqsp_example.py similarity index 100% rename from examples/llm_plus_gnn/test_nvtx_webqsp.py rename to examples/llm_plus_gnn/nvtx_webqsp_example.py From e63af63b83195e78b1b1e9ccb3beffc87a1d10ae Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 19:43:41 -0700 Subject: [PATCH 669/752] changelog 0 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index acfe87a1cd70..fd1e9cfb78fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added `profiler.nvtxit` with some examples ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `loader.RagQueryLoader` with Remote Backend Example ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `data.LargeGraphIndexer` ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `nn.models.GRetriever` ([#9167](https://github.com/pyg-team/pytorch_geometric/pull/9167)) From 05269ca3e9fce96b6317af16af357349f3058295 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 04:05:35 -0700 Subject: [PATCH 670/752] docstrings --- .../{ => nvtx_examples}/nvtx_rag_backend_example.py | 0 .../{ => nvtx_examples}/nvtx_uwebqsp_example.py | 0 .../{ => nvtx_examples}/nvtx_webqsp_example.py | 0 examples/llm_plus_gnn/nvtx_run.sh | 2 +- torch_geometric/profile/nvtx.py | 11 +++++++++++ 5 files changed, 12 insertions(+), 1 deletion(-) rename examples/llm_plus_gnn/{ => nvtx_examples}/nvtx_rag_backend_example.py (100%) rename examples/llm_plus_gnn/{ => nvtx_examples}/nvtx_uwebqsp_example.py (100%) rename examples/llm_plus_gnn/{ => nvtx_examples}/nvtx_webqsp_example.py (100%) diff --git a/examples/llm_plus_gnn/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py similarity index 100% rename from examples/llm_plus_gnn/nvtx_rag_backend_example.py rename to examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py diff --git a/examples/llm_plus_gnn/nvtx_uwebqsp_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_uwebqsp_example.py similarity index 100% rename from examples/llm_plus_gnn/nvtx_uwebqsp_example.py rename to examples/llm_plus_gnn/nvtx_examples/nvtx_uwebqsp_example.py diff --git a/examples/llm_plus_gnn/nvtx_webqsp_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py similarity index 100% rename from examples/llm_plus_gnn/nvtx_webqsp_example.py rename to examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_run.sh index b40c32abb0ea..4c6fce7c8224 100755 --- a/examples/llm_plus_gnn/nvtx_run.sh +++ b/examples/llm_plus_gnn/nvtx_run.sh @@ -24,4 +24,4 @@ python_file=$(basename "$1") # Run nsys profile on the Python file nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cuda-memory-usage true --cudabacktrace all --force-overwrite true --output=profile_${python_file%.py} python "$1" -echo "Profile data saved as profile_${python_file%.py}.nsys-rep" \ No newline at end of file +echo "Profile data saved as profile_${python_file%.py}.nsys-rep" diff --git a/torch_geometric/profile/nvtx.py b/torch_geometric/profile/nvtx.py index 122ddfad916f..8dbce375ae5a 100644 --- a/torch_geometric/profile/nvtx.py +++ b/torch_geometric/profile/nvtx.py @@ -24,6 +24,17 @@ def end_cuda_profile(prev_state: bool): def nvtxit(name: Optional[str] = None, n_warmups: int = 0, n_iters: Optional[int] = None): + """Enables NVTX profiling for a function. + + Args: + name (Optional[str], optional): Name to give the reference frame for + the function being wrapped. Defaults to the name of the + function in code. + n_warmups (int, optional): Number of iters to call that function + before starting. Defaults to 0. + n_iters (Optional[int], optional): Number of iters of that function to + record. Defaults to all of them. + """ def nvtx(func): nonlocal name From 11293ff19425deab2157d2648e1375c6c6183ec2 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 23 Aug 2024 17:25:49 -0700 Subject: [PATCH 671/752] nvtx performance additons --- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 95 +++++++++++++++++++-- 1 file changed, 88 insertions(+), 7 deletions(-) diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb index d0f6cc3c2e4d..c3f8e37873a6 100644 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -27,7 +27,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph." + "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph, and conduct some experiments to evaluate its runtime performance." ] }, { @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -49,7 +49,7 @@ "" ] }, - "execution_count": 37, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -20285,11 +20285,92 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "## Evaluating Runtime Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pytorch Geometric provides multiple methods for evalutaing runtime performance. In this notebook, we utilize NVTX to profile the different components of our RAG Query Loader." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The method `nvtxit` allows for profiling the utilization and timings of any methods that get wrapped by it in a Python script." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To see an example of this, check out `nvtx_examples/nvtx_rag_backend_example.py`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This script mirrors this notebook's functionality, but notably, it includes the following code snippet:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "# Patch FeatureStore and GraphStore\n", + "\n", + "SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_nodes)\n", + "SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_edges)\n", + "SentenceTransformerFeatureStore.load_subgraph = nvtxit()(SentenceTransformerFeatureStore.load_subgraph)\n", + "NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()(NeighborSamplingRAGGraphStore.sample_subgraph)\n", + "rag_loader.RAGQueryLoader.query = nvtxit()(rag_loader.RAGQueryLoader.query)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, this snippet wraps the methods of FeatureStore, GraphStore, and the Query method from QueryLoader so that it will be recognized as a unique frame in NVTX." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This can be executed by the included shell script `nvtx_run.sh`:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "...\n", + "\n", + "# Get the base name of the Python file\n", + "python_file=$(basename \"$1\")\n", + "\n", + "# Run nsys profile on the Python file\n", + "nsys profile -c cudaProfilerApi --capture-range-end repeat -t cuda,nvtx,osrt,cudnn,cublas --cuda-memory-usage true --cudabacktrace all --force-overwrite true --output=profile_${python_file%.py} python \"$1\"\n", + "\n", + "echo \"Profile data saved as profile_${python_file%.py}.nsys-rep\"\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The generated resulting `.nsys-rep` file can be visualized using tools like Nsight Systems or Nsight Compute, that can show the relative timings of the FeatureStore, GraphStore, and QueryLoader methods." + ] } ], "metadata": { From 530405c1d1d662b2ec46b9662b445cc3051f82d4 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:37:13 -0700 Subject: [PATCH 672/752] Changes to allow for multiprocessing in hallucination detection --- examples/llm_plus_gnn/g_retriever.py | 73 +++++++++++++++------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 7ef79cc47b8f..1bae976625e6 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -16,6 +16,8 @@ import torch from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm +import torch.nn as nn +from typing import Callable, List, Type, Dict, Any from torch_geometric import seed_everything from torch_geometric.data import Dataset @@ -23,9 +25,10 @@ from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM +import multiprocessing as mp - -def detect_hallucinate(pred, label): +def _detect_hallucinate(inp): + pred, label = inp try: split_pred = pred.split('[/s]')[0].strip().split('|') correct_hit = len(re.findall(split_pred[0], label)) > 0 @@ -284,7 +287,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=1, drop_last=False, + loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, pin_memory=True, shuffle=False) if tiny_llama: pure_llm = LLM( @@ -317,8 +320,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, "Checking pretrained LLM vs trained GNN+LLM for hallucinations..." # noqa ) for i, batch in enumerate(tqdm(loader)): - question = batch.question[0] - correct_answer = batch.label[0] + question = batch.question + correct_answer = batch.label if skip_pretrained_LLM: pure_llm_pred = None pure_llm_hallucinates = False @@ -329,18 +332,19 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [(pure_llm_pred, pure_llm_hallucinates)] + untuned_llm_save_list += [tup for tup in zip(pure_llm_pred, pure_llm_hallucinates)] - gnn_llm_pred = gnn_llm_preds[i] + gnn_llm_pred = gnn_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [(gnn_llm_pred, gnn_llm_hallucinates)] - - if gnn_llm_hallucinates == "skip" or pure_llm_hallucinates == "skip": # noqa - # skipping when hallucination is hard to eval - continue - gnn_llm_hallucin_sum += int(gnn_llm_hallucinates) - pure_llm_hallucin_sum += int(pure_llm_hallucinates) + gnn_save_list += [tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates)] + + for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, pure_llm_hallucinates): + if gnn_llm_hal == "skip" or pure_llm_hal == "skip": # noqa + # skipping when hallucination is hard to eval + continue + gnn_llm_hallucin_sum += int(gnn_llm_hal) + pure_llm_hallucin_sum += int(pure_llm_hal) if not skip_pretrained_LLM: print("Total Pure LLM Hallucinations:", pure_llm_hallucin_sum) print("Total GNN+LLM Hallucinations:", gnn_llm_hallucin_sum) @@ -391,30 +395,31 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_preds += out['pred'] print("Final comparison between all models...") for i, batch in enumerate(tqdm(loader)): - question = batch.question[0] - correct_answer = batch.label[0] - gnn_llm_pred, gnn_llm_hallucinates = gnn_save_list[i] - untuned_llm_pred, untuned_llm_hallucinates = untuned_llm_save_list[i] + question = batch.question + correct_answer = batch.label + gnn_llm_pred, gnn_llm_hallucinates = list(zip(*gnn_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) + untuned_llm_pred, untuned_llm_hallucinates = list(zip(*untuned_llm_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm_preds[i] + pure_llm_pred = pure_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) - if pure_llm_hallucinates == "skip": - continue - trained_llm_hallucin_sum += int(pure_llm_hallucinates) - if skip_pretrained_LLM: - # we did not check the untrained LLM, so do not decide to demo - # based on this. - untuned_llm_hallucinates = True - if untuned_llm_hallucinates and pure_llm_hallucinates and not gnn_llm_hallucinates: # noqa - final_prnt_str += "Prompt: '" + question + "'\n" - final_prnt_str += "Label: '" + correct_answer + "'\n" - if not skip_pretrained_LLM: - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred + "'\n" # noqa - final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred + "'\n" - final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred + "'\n" - final_prnt_str += "\n" + "#" * 20 + "\n\n" + for j in range(len(gnn_llm_pred)): + if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[j] == "skip" or pure_llm_hallucinates[j] == "skip": + continue + trained_llm_hallucin_sum += int(pure_llm_hallucinates[j]) + if skip_pretrained_LLM: + # we did not check the untrained LLM, so do not decide to demo + # based on this. + untuned_llm_hallucinates = True + if untuned_llm_hallucinates[j] and pure_llm_hallucinates[j] and not gnn_llm_hallucinates[j]: # noqa + final_prnt_str += "Prompt: '" + question[j] + "'\n" + final_prnt_str += "Label: '" + correct_answer[j] + "'\n" + if not skip_pretrained_LLM: + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[j] + "'\n" # noqa + final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[j] + "'\n" + final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred[j] + "'\n" + final_prnt_str += "\n" + "#" * 20 + "\n\n" if not skip_pretrained_LLM: print("Total untuned LLM Hallucinations:", untuned_llm_hallucin_sum) print("Total tuned LLM Hallucinations:", trained_llm_hallucin_sum) From 35b8dd006092c90cd7a24c1ae65e551d3b84d638 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 15 Aug 2024 16:54:15 -0700 Subject: [PATCH 673/752] restore changes that were for benchmarking and experiments --- .../llm_plus_gnn/benchmark_model_archs.py | 37 +++++ .../llm_plus_gnn/benchmark_model_gnn_enc.py | 39 +++++ examples/llm_plus_gnn/g_retriever.py | 147 +++++++++++++++++- torch_geometric/nn/models/g_retriever.py | 5 +- 4 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 examples/llm_plus_gnn/benchmark_model_archs.py create mode 100644 examples/llm_plus_gnn/benchmark_model_gnn_enc.py diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py new file mode 100644 index 000000000000..f73d16b7ed9d --- /dev/null +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -0,0 +1,37 @@ +# %% +from g_retriever import benchmark_models, get_loss, inference_step +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.models import GRetriever, MLP, GAT + +# %% +hidden_channels = 1024 +num_gnn_layers = 1 +lr=1e-5 +epochs=2 +batch_size=8 +eval_batch_size=16 + +# %% +ds = UpdatedWebQSPDataset('benchmark_archs') + +# %% +model_names = [] +model_classes = [] +model_kwargs = [] +model_type = ["GAT", "MLP"] +models = {"GAT": GAT, "MLP": MLP} +num_layers = [1, 2, 4, 8, 16, 32] +for m_type in model_type: + for n_layer in num_layers: + model_names.append(f"{m_type}_{n_layer}") + model_classes.append(GRetriever) + kwargs = dict(gnn_hidden_channels=hidden_channels, num_gnn_layers=n_layer, gnn_to_use=models[m_type]) + model_kwargs.append(kwargs) + +# %% +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) + +# %% + + + diff --git a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py new file mode 100644 index 000000000000..db1210b70ed4 --- /dev/null +++ b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py @@ -0,0 +1,39 @@ +# %% +from g_retriever import benchmark_models, get_loss, inference_step +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.models import GRetriever, MLP, GAT + +# %% +hidden_channels = 1024 +num_gnn_layers = 1 +lr=1e-5 +epochs=2 +batch_size=8 +eval_batch_size=16 + +# %% +ds = UpdatedWebQSPDataset('benchmark_archs') + +# %% +model_names = [] +model_classes = [] +model_kwargs = [] +model_type = ["GAT"] +models = {"GAT": GAT, "MLP": MLP} +num_layers = [4] +num_tokens = [4, 16] +for m_type in model_type: + for n_layer in num_layers: + for n_tokens in num_tokens + model_names.append(f"{m_type}_{n_layer}_{n_tokens}") + model_classes.append(GRetriever) + kwargs = dict(gnn_hidden_channels=hidden_channels*n_tokens, num_gnn_layers=n_layer, gnn_to_use=models[m_type], gnn_out_channels=hidden_channels*n_tokens, mlp_out_tokens=n_tokens) + model_kwargs.append(kwargs) + +# %% +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) + +# %% + + + diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1bae976625e6..dfbc880d48f0 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -92,6 +92,9 @@ def compute_accuracy(eval_output) -> float: return hit +def compute_n_parameters(model: torch.nn.Module) -> int: + return sum([p.numel() for p in model.parameters() if p.requires_grad]) + def save_params_dict(model, save_path): state_dict = model.state_dict() @@ -129,9 +132,10 @@ def inference_step(model, batch, model_save_name): batch.desc) + def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=None, dataset=None, - checkpointing=False, tiny_llama=False): + checkpointing=False, tiny_llama=False, model_save_name=None): def adjust_learning_rate(param_group, LR, epoch): # Decay the learning rate with half-cycle cosine after warmup min_lr = 5e-6 @@ -181,10 +185,11 @@ def adjust_learning_rate(param_group, LR, epoch): model = GRetriever(llm_to_use="meta-llama/Llama-2-7b-chat-hf", gnn_hidden_channels=hidden_channels, num_gnn_layers=num_gnn_layers) - if num_gnn_layers is not None: - model_save_name = "gnn_llm" - else: - model_save_name = "llm" + if model_save_name is None: + if num_gnn_layers is not None: + model_save_name = "gnn_llm" + else: + model_save_name = "llm" # Step 3 Set Optimizer params = [p for _, p in model.named_parameters() if p.requires_grad] @@ -277,6 +282,138 @@ def adjust_learning_rate(param_group, LR, epoch): print("Done!") return prep_time, dataset, eval_output +def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): + model_save_list = [] + model_preds = [] + for out in outs: + model_preds += out['pred'] + for i, batch in enumerate(loader): + correct_answer = batch.label + + model_pred = model_preds[i*eval_batch_size:(i+1)*eval_batch_size] + model_hallucinates = detect_hallucinate(model_pred, + correct_answer) + model_save_list += [tup for tup in zip(model_pred, model_hallucinates)] + return model_save_list + + +def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True, force: bool = False, root_dir='.'): + + model_log: Dict[str, Dict[str, Any]] = dict() + idx_split = dataset.split_idxs + test_dataset = [dataset[i] for i in idx_split['test']] + loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, + pin_memory=True, shuffle=False) + + if not skip_LLMs: + if tiny_llama: + pure_llm = LLM( + model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_params=1, + ) + else: + pure_llm = LLM(model_name="llama2-7b", num_params=7) + + if not path.exists(root_dir +"/pure_llm_model_log.pt"): + model_log["pure_llm"] = dict() + + pure_preds = [] + for batch in tqdm(loader): + pure_llm_preds = pure_llm.inference(batch.question, batch.desc, max_tokens=256) + pure_preds += pure_llm_preds + pure_preds = [{"pred": pred} for pred in pure_preds] + + model_log["pure_llm"]["preds"] = pure_preds + model_log["pure_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(pure_preds, loader, eval_batch_size) + model_log["pure_llm"]["n_params"] = compute_n_parameters(pure_llm) + torch.save(model_log["pure_llm"], root_dir+"/pure_llm_model_log.pt") + else: + model_log["pure_llm"] = torch.load(root_dir+"/pure_llm_model_log.pt") + + # LORA + if not path.exists(root_dir+"/tuned_llm_model_log.pt"): + model_log["tuned_llm"] = dict() + since = time.time() + gc.collect() + prep_time, _, lora_eval_outs = train(since, epochs, None, None, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=pure_llm, dataset=dataset) + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() + e2e_time = round(time.time() - since, 2) + model_log["tuned_llm"]["prep_time"] = prep_time + model_log["tuned_llm"]["e2e_time"] = e2e_time + model_log["tuned_llm"]["eval_output"] = lora_eval_outs + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") + + model_log["tuned_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(lora_eval_outs, loader, eval_batch_size) + model_log["tuned_llm"]["n_params"] = compute_n_parameters(pure_llm) + torch.save(model_log["tuned_llm"], root_dir+"/tuned_llm_model_log.pt") + else: + model_log["tuned_llm"] = torch.load(root_dir+"/tuned_llm_model_log.pt") + + del pure_llm + gc.collect() + + # All other models + for name, Model, kwargs in zip(model_names, models, model_kwargs): + model_log[name] = dict() + train_model = True + if path.exists(root_dir+f"/{name}.pt") and not force: + print(f"Model {name} appears to already exist.") + print("Would you like to retrain?") + train_model = str(input("(y/n):")).lower() == "y" + + if train_model: + since = time.time() + gc.collect() + model = Model(**kwargs) + prep_time, _, model_eval_outs = train( + since=since, num_epochs=epochs, hidden_channels=None, num_gnn_layers=None, + batch_size=batch_size, eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, + inference_fn=inference_fn, checkpointing=checkpointing, + tiny_llama=tiny_llama, dataset=dataset, model_save_name=root_dir+'/'+name, model=model) + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + gc.collect() + e2e_time = round(time.time() - since, 2) + model_log[name]["prep_time"] = prep_time + model_log[name]["e2e_time"] = e2e_time + model_log[name]["eval_output"] = model_eval_outs + print("E2E time (e2e_time) =", e2e_time, "seconds") + print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") + model_log[name]["n_params"] = compute_n_parameters(model) + del model + gc.collect() + else: + model_eval_outs = torch.load(root_dir+f"/{name}_eval_outs.pt") + + # Calculate Hallucinations + skip_hallucination_detection = False + + if path.exists(root_dir+f"/{name}_model_log.pt") and not force: + print(f"Saved outputs for {name} have been found.") + print("Would you like to redo?") + user_input = str(input("(y/n):")).lower() + skip_hallucination_detection = user_input != "y" + + + if not skip_hallucination_detection: + model_save_list = _eval_hallucinations_on_loader(model_eval_outs, loader, eval_batch_size) + + model_log[name]["hallucinates_list"] = model_save_list + torch.save(model_log[name], root_dir+f"/{name}_model_log.pt") + else: + model_log[name]["hallucinates_list"] = torch.load(root_dir+f"/{name}_model_log.pt")["hallucinates_list"] + + hal_dict = {k: [tup[1] for tup in v["hallucinates_list"]] for (k, v) in model_log.items()} + hallucinates_df = pd.DataFrame(hal_dict).astype(str) + hallucinates_df = hallucinates_df.apply(pd.Series.value_counts).transpose() + hallucinates_df['e2e_time'] = pd.Series({k: v.get('e2e_time') for (k, v) in model_log.items()}) + hallucinates_df['n_params'] = pd.Series({k: v.get('n_params') for (k, v) in model_log.items()}) + print(hallucinates_df) + hallucinates_df.to_csv(root_dir+"/hallucinates_df.csv", index=False) + def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, eval_batch_size, loss_fn, inference_fn, diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 573434fc00fa..0a9182bdf858 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -61,6 +61,7 @@ def __init__( num_gnn_heads: int = 4, mlp_hidden_dim: int = 2048, mlp_out_dim: int = 4096, + mlp_out_tokens: int = 1, ) -> None: super().__init__() self.llm_to_use = LLM(llm_to_use, num_llm_params, llm_dtype) @@ -115,9 +116,11 @@ def __init__( self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), - nn.Linear(mlp_hidden_dim, mlp_out_dim), + nn.Linear(mlp_hidden_dim, mlp_out_dim*mlp_out_tokens), ).to(self.llm_device) + self.mlp_out_tokens = mlp_out_tokens + self.word_embedding = self.llm_to_use.word_embedding def encode_graphs(self, node_feat, edge_index, edge_attr, batch): From 46a7d7a8bde77f702ce5c41ccdbdf6f9cee6980a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 16 Aug 2024 20:00:22 -0700 Subject: [PATCH 674/752] changelog 0 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd1e9cfb78fd..8f7b42ff0f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added various GRetriever Architecture Benchmarking examples ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `profiler.nvtxit` with some examples ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `loader.RagQueryLoader` with Remote Backend Example ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) - Added `data.LargeGraphIndexer` ([#9597](https://github.com/pyg-team/pytorch_geometric/pull/9597)) From 85bfeaa9420c5bb27ceaeedc96d42fd7ef6ebb7a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 21 Aug 2024 04:34:22 -0700 Subject: [PATCH 675/752] docstrings --- .../llm_plus_gnn/benchmark_model_archs.py | 27 +-- .../llm_plus_gnn/benchmark_model_gnn_enc.py | 25 +-- examples/llm_plus_gnn/g_retriever.py | 184 ++++++++++++------ 3 files changed, 158 insertions(+), 78 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index f73d16b7ed9d..55743586d9f1 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -1,15 +1,18 @@ +"""Used to benchmark the performance of an untuned/fine tuned LLM against +GRetriever with various architectures and layer depths.""" # %% from g_retriever import benchmark_models, get_loss, inference_step + from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.models import GRetriever, MLP, GAT +from torch_geometric.nn.models import GAT, MLP, GRetriever # %% hidden_channels = 1024 num_gnn_layers = 1 -lr=1e-5 -epochs=2 -batch_size=8 -eval_batch_size=16 +lr = 1e-5 +epochs = 2 +batch_size = 8 +eval_batch_size = 16 # %% ds = UpdatedWebQSPDataset('benchmark_archs') @@ -20,18 +23,18 @@ model_kwargs = [] model_type = ["GAT", "MLP"] models = {"GAT": GAT, "MLP": MLP} -num_layers = [1, 2, 4, 8, 16, 32] +num_layers = [1, 4, 16] for m_type in model_type: for n_layer in num_layers: model_names.append(f"{m_type}_{n_layer}") model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels, num_gnn_layers=n_layer, gnn_to_use=models[m_type]) + kwargs = dict(gnn_hidden_channels=hidden_channels, + num_gnn_layers=n_layer, gnn_to_use=models[m_type]) model_kwargs.append(kwargs) # %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) - -# %% - - +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, + batch_size, eval_batch_size, get_loss, inference_step, + skip_LLMs=False, tiny_llama=True, force=True) +# TODO Argparse options diff --git a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py index db1210b70ed4..3b65e0043ef6 100644 --- a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py +++ b/examples/llm_plus_gnn/benchmark_model_gnn_enc.py @@ -1,15 +1,16 @@ # %% from g_retriever import benchmark_models, get_loss, inference_step + from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.models import GRetriever, MLP, GAT +from torch_geometric.nn.models import GAT, MLP, GRetriever # %% hidden_channels = 1024 num_gnn_layers = 1 -lr=1e-5 -epochs=2 -batch_size=8 -eval_batch_size=16 +lr = 1e-5 +epochs = 2 +batch_size = 8 +eval_batch_size = 16 # %% ds = UpdatedWebQSPDataset('benchmark_archs') @@ -24,16 +25,18 @@ num_tokens = [4, 16] for m_type in model_type: for n_layer in num_layers: - for n_tokens in num_tokens + for n_tokens in num_tokens: model_names.append(f"{m_type}_{n_layer}_{n_tokens}") model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels*n_tokens, num_gnn_layers=n_layer, gnn_to_use=models[m_type], gnn_out_channels=hidden_channels*n_tokens, mlp_out_tokens=n_tokens) + kwargs = dict(gnn_hidden_channels=hidden_channels * n_tokens, + num_gnn_layers=n_layer, gnn_to_use=models[m_type], + gnn_out_channels=hidden_channels * n_tokens, + mlp_out_tokens=n_tokens) model_kwargs.append(kwargs) # %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=True, force=True) +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, + batch_size, eval_batch_size, get_loss, inference_step, + skip_LLMs=False, tiny_llama=True, force=True) # %% - - - diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index dfbc880d48f0..e642c07f5a57 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -8,16 +8,17 @@ import argparse import gc import math +import multiprocessing as mp import re import time from os import path +from typing import Any, Callable, Dict, List, Type import pandas as pd import torch +import torch.nn as nn from torch.nn.utils import clip_grad_norm_ from tqdm import tqdm -import torch.nn as nn -from typing import Callable, List, Type, Dict, Any from torch_geometric import seed_everything from torch_geometric.data import Dataset @@ -25,7 +26,7 @@ from torch_geometric.loader import DataLoader from torch_geometric.nn.models import GRetriever from torch_geometric.nn.nlp import LLM -import multiprocessing as mp + def _detect_hallucinate(inp): pred, label = inp @@ -49,6 +50,7 @@ def detect_hallucinate(pred_batch, label_batch): res = p.map(_detect_hallucinate, zip(pred_batch, label_batch)) return res + def compute_accuracy(eval_output) -> float: df = pd.concat([pd.DataFrame(d) for d in eval_output]) all_hit = [] @@ -92,6 +94,7 @@ def compute_accuracy(eval_output) -> float: return hit + def compute_n_parameters(model: torch.nn.Module) -> int: return sum([p.numel() for p in model.parameters() if p.requires_grad]) @@ -132,7 +135,6 @@ def inference_step(model, batch, model_save_name): batch.desc) - def train(since, num_epochs, hidden_channels, num_gnn_layers, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=None, dataset=None, checkpointing=False, tiny_llama=False, model_save_name=None): @@ -282,6 +284,7 @@ def adjust_learning_rate(param_group, LR, epoch): print("Done!") return prep_time, dataset, eval_output + def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): model_save_list = [] model_preds = [] @@ -290,20 +293,52 @@ def _eval_hallucinations_on_loader(outs, loader, eval_batch_size): for i, batch in enumerate(loader): correct_answer = batch.label - model_pred = model_preds[i*eval_batch_size:(i+1)*eval_batch_size] - model_hallucinates = detect_hallucinate(model_pred, - correct_answer) + model_pred = model_preds[i * eval_batch_size:(i + 1) * eval_batch_size] + model_hallucinates = detect_hallucinate(model_pred, correct_answer) model_save_list += [tup for tup in zip(model_pred, model_hallucinates)] return model_save_list -def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], model_kwargs: List[Dict[str, Any]], dataset: Dataset, lr: float, epochs: int, batch_size: int, eval_batch_size: int, loss_fn: Callable, inference_fn: Callable, skip_LLMs: bool = True, tiny_llama: bool = False, checkpointing: bool = True, force: bool = False, root_dir='.'): - +def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], + model_kwargs: List[Dict[str, Any]], dataset: Dataset, + lr: float, epochs: int, batch_size: int, + eval_batch_size: int, loss_fn: Callable, + inference_fn: Callable, skip_LLMs: bool = True, + tiny_llama: bool = False, checkpointing: bool = True, + force: bool = False, root_dir='.'): + """Utility function for creating a model benchmark for GRetriever that + grid searches over hyperparameters. Produces a DataFrame containing + metrics for each model. + + Args: + models (List[Type[nn.Module]]): Models to be benchmarked. + model_names (List[str]): Name of save files for model checkpoints + model_kwargs (List[Dict[str, Any]]): Parameters to use for each + particular model. + dataset (Dataset): Input dataset to train on. + lr (float): Learning rate + epochs (int): Number of epochs + batch_size (int): Batch size for training + eval_batch_size (int): Batch size for eval. Also determines + hallucination detection concurrancy. + loss_fn (Callable): Loss function + inference_fn (Callable): Inference function + skip_LLMs (bool, optional): Whether to skip LLM-only runs. + Defaults to True. + tiny_llama (bool, optional): Whether to use tiny llama as LLM. + Defaults to False. + checkpointing (bool, optional): Whether to checkpoint models. + Defaults to True. + force (bool, optional): Whether to rerun already existing results. + Defaults to False. + root_dir (str, optional): Dir to save results and checkpoints in. + Defaults to '.'. + """ model_log: Dict[str, Dict[str, Any]] = dict() idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] - loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, - pin_memory=True, shuffle=False) + loader = DataLoader(test_dataset, batch_size=eval_batch_size, + drop_last=False, pin_memory=True, shuffle=False) if not skip_LLMs: if tiny_llama: @@ -314,28 +349,37 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode else: pure_llm = LLM(model_name="llama2-7b", num_params=7) - if not path.exists(root_dir +"/pure_llm_model_log.pt"): + if not path.exists(root_dir + "/pure_llm_model_log.pt"): model_log["pure_llm"] = dict() - + pure_preds = [] for batch in tqdm(loader): - pure_llm_preds = pure_llm.inference(batch.question, batch.desc, max_tokens=256) + pure_llm_preds = pure_llm.inference(batch.question, batch.desc, + max_tokens=256) pure_preds += pure_llm_preds pure_preds = [{"pred": pred} for pred in pure_preds] model_log["pure_llm"]["preds"] = pure_preds - model_log["pure_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(pure_preds, loader, eval_batch_size) + model_log["pure_llm"]["hallucinates_list"] = \ + _eval_hallucinations_on_loader(pure_preds, loader, + eval_batch_size) model_log["pure_llm"]["n_params"] = compute_n_parameters(pure_llm) - torch.save(model_log["pure_llm"], root_dir+"/pure_llm_model_log.pt") + torch.save(model_log["pure_llm"], + root_dir + "/pure_llm_model_log.pt") else: - model_log["pure_llm"] = torch.load(root_dir+"/pure_llm_model_log.pt") + model_log["pure_llm"] = \ + torch.load(root_dir+"/pure_llm_model_log.pt") # LORA - if not path.exists(root_dir+"/tuned_llm_model_log.pt"): + if not path.exists(root_dir + "/tuned_llm_model_log.pt"): model_log["tuned_llm"] = dict() since = time.time() gc.collect() - prep_time, _, lora_eval_outs = train(since, epochs, None, None, batch_size, eval_batch_size, lr, loss_fn, inference_fn, model=pure_llm, dataset=dataset) + prep_time, _, lora_eval_outs = train(since, epochs, None, None, + batch_size, eval_batch_size, + lr, loss_fn, inference_fn, + model=pure_llm, + dataset=dataset) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -346,33 +390,39 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode print("E2E time (e2e_time) =", e2e_time, "seconds") print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") - model_log["tuned_llm"]["hallucinates_list"] = _eval_hallucinations_on_loader(lora_eval_outs, loader, eval_batch_size) + model_log["tuned_llm"]["hallucinates_list"] = \ + _eval_hallucinations_on_loader(lora_eval_outs, loader, + eval_batch_size) model_log["tuned_llm"]["n_params"] = compute_n_parameters(pure_llm) - torch.save(model_log["tuned_llm"], root_dir+"/tuned_llm_model_log.pt") + torch.save(model_log["tuned_llm"], + root_dir + "/tuned_llm_model_log.pt") else: - model_log["tuned_llm"] = torch.load(root_dir+"/tuned_llm_model_log.pt") - + model_log["tuned_llm"] = \ + torch.load(root_dir+"/tuned_llm_model_log.pt") + del pure_llm gc.collect() - + # All other models for name, Model, kwargs in zip(model_names, models, model_kwargs): model_log[name] = dict() train_model = True - if path.exists(root_dir+f"/{name}.pt") and not force: + if path.exists(root_dir + f"/{name}.pt") and not force: print(f"Model {name} appears to already exist.") print("Would you like to retrain?") train_model = str(input("(y/n):")).lower() == "y" - + if train_model: since = time.time() gc.collect() model = Model(**kwargs) prep_time, _, model_eval_outs = train( - since=since, num_epochs=epochs, hidden_channels=None, num_gnn_layers=None, - batch_size=batch_size, eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, + since=since, num_epochs=epochs, hidden_channels=None, + num_gnn_layers=None, batch_size=batch_size, + eval_batch_size=eval_batch_size, lr=lr, loss_fn=loss_fn, inference_fn=inference_fn, checkpointing=checkpointing, - tiny_llama=tiny_llama, dataset=dataset, model_save_name=root_dir+'/'+name, model=model) + tiny_llama=tiny_llama, dataset=dataset, + model_save_name=root_dir + '/' + name, model=model) torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() gc.collect() @@ -386,33 +436,43 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], mode del model gc.collect() else: - model_eval_outs = torch.load(root_dir+f"/{name}_eval_outs.pt") - + model_eval_outs = torch.load(root_dir + f"/{name}_eval_outs.pt") + # Calculate Hallucinations skip_hallucination_detection = False - if path.exists(root_dir+f"/{name}_model_log.pt") and not force: + if path.exists(root_dir + f"/{name}_model_log.pt") and not force: print(f"Saved outputs for {name} have been found.") print("Would you like to redo?") user_input = str(input("(y/n):")).lower() skip_hallucination_detection = user_input != "y" - if not skip_hallucination_detection: - model_save_list = _eval_hallucinations_on_loader(model_eval_outs, loader, eval_batch_size) - + model_save_list = _eval_hallucinations_on_loader( + model_eval_outs, loader, eval_batch_size) + model_log[name]["hallucinates_list"] = model_save_list - torch.save(model_log[name], root_dir+f"/{name}_model_log.pt") + torch.save(model_log[name], root_dir + f"/{name}_model_log.pt") else: - model_log[name]["hallucinates_list"] = torch.load(root_dir+f"/{name}_model_log.pt")["hallucinates_list"] - - hal_dict = {k: [tup[1] for tup in v["hallucinates_list"]] for (k, v) in model_log.items()} + model_log[name]["hallucinates_list"] = \ + torch.load( + root_dir+f"/{name}_model_log.pt" + )["hallucinates_list"] + + hal_dict = { + k: [tup[1] for tup in v["hallucinates_list"]] + for (k, v) in model_log.items() + } hallucinates_df = pd.DataFrame(hal_dict).astype(str) hallucinates_df = hallucinates_df.apply(pd.Series.value_counts).transpose() - hallucinates_df['e2e_time'] = pd.Series({k: v.get('e2e_time') for (k, v) in model_log.items()}) - hallucinates_df['n_params'] = pd.Series({k: v.get('n_params') for (k, v) in model_log.items()}) + hallucinates_df['e2e_time'] = pd.Series( + {k: v.get('e2e_time') + for (k, v) in model_log.items()}) + hallucinates_df['n_params'] = pd.Series( + {k: v.get('n_params') + for (k, v) in model_log.items()}) print(hallucinates_df) - hallucinates_df.to_csv(root_dir+"/hallucinates_df.csv", index=False) + hallucinates_df.to_csv(root_dir + "/hallucinates_df.csv", index=False) def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, @@ -424,8 +484,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, idx_split = dataset.split_idxs test_dataset = [dataset[i] for i in idx_split['test']] # batch size 1 loader for simplicity - loader = DataLoader(test_dataset, batch_size=eval_batch_size, drop_last=False, - pin_memory=True, shuffle=False) + loader = DataLoader(test_dataset, batch_size=eval_batch_size, + drop_last=False, pin_memory=True, shuffle=False) if tiny_llama: pure_llm = LLM( model_name="TinyLlama/TinyLlama-1.1B-Chat-v0.1", @@ -469,14 +529,20 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) - untuned_llm_save_list += [tup for tup in zip(pure_llm_pred, pure_llm_hallucinates)] + untuned_llm_save_list += [ + tup for tup in zip(pure_llm_pred, pure_llm_hallucinates) + ] - gnn_llm_pred = gnn_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] + gnn_llm_pred = gnn_llm_preds[i * eval_batch_size:(i + 1) * + eval_batch_size] gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, correct_answer) - gnn_save_list += [tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates)] + gnn_save_list += [ + tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates) + ] - for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, pure_llm_hallucinates): + for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, + pure_llm_hallucinates): if gnn_llm_hal == "skip" or pure_llm_hal == "skip": # noqa # skipping when hallucination is hard to eval continue @@ -534,27 +600,35 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, for i, batch in enumerate(tqdm(loader)): question = batch.question correct_answer = batch.label - gnn_llm_pred, gnn_llm_hallucinates = list(zip(*gnn_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) - untuned_llm_pred, untuned_llm_hallucinates = list(zip(*untuned_llm_save_list[i*eval_batch_size:(i+1)*eval_batch_size])) + gnn_llm_pred, gnn_llm_hallucinates = list( + zip(*gnn_save_list[i * eval_batch_size:(i + 1) * eval_batch_size])) + untuned_llm_pred, untuned_llm_hallucinates = list( + zip(*untuned_llm_save_list[i * eval_batch_size:(i + 1) * + eval_batch_size])) if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa continue - pure_llm_pred = pure_llm_preds[i*eval_batch_size:(i+1)*eval_batch_size] + pure_llm_pred = pure_llm_preds[i * eval_batch_size:(i + 1) * + eval_batch_size] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) for j in range(len(gnn_llm_pred)): - if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[j] == "skip" or pure_llm_hallucinates[j] == "skip": + if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[ + j] == "skip" or pure_llm_hallucinates[j] == "skip": continue trained_llm_hallucin_sum += int(pure_llm_hallucinates[j]) if skip_pretrained_LLM: # we did not check the untrained LLM, so do not decide to demo # based on this. untuned_llm_hallucinates = True - if untuned_llm_hallucinates[j] and pure_llm_hallucinates[j] and not gnn_llm_hallucinates[j]: # noqa + if untuned_llm_hallucinates[j] and pure_llm_hallucinates[ + j] and not gnn_llm_hallucinates[j]: # noqa final_prnt_str += "Prompt: '" + question[j] + "'\n" final_prnt_str += "Label: '" + correct_answer[j] + "'\n" if not skip_pretrained_LLM: - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[j] + "'\n" # noqa - final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[j] + "'\n" + final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[ + j] + "'\n" # noqa + final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[ + j] + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred[j] + "'\n" final_prnt_str += "\n" + "#" * 20 + "\n\n" if not skip_pretrained_LLM: From 23872a3024b5976ce89485e9c6aa7ba755f62b0a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Sat, 24 Aug 2024 01:33:54 -0700 Subject: [PATCH 676/752] passs in arbitrary dataset path --- .../llm_plus_gnn/benchmark_model_archs.py | 57 ++++++++++++++++--- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index 55743586d9f1..651c95acdbd1 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -2,20 +2,61 @@ GRetriever with various architectures and layer depths.""" # %% from g_retriever import benchmark_models, get_loss, inference_step - +import argparse from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever +import torch + # %% -hidden_channels = 1024 -num_gnn_layers = 1 -lr = 1e-5 -epochs = 2 -batch_size = 8 -eval_batch_size = 16 +parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") +parser.add_argument("--hidden_channels", type=int, default=1024) +parser.add_argument("--learning_rate", type=float, default=1e-5) +parser.add_argument("--epochs", type=int, default=2) +parser.add_argument("--batch_size", type=int, default=8) +parser.add_argument("--eval_batch_size", type=int, default=16) +parser.add_argument("--tiny_llama", store_true=True) + +parser.add_argument("--custom_dataset", store_true=True) +parser.add_argument("--dataset_path", type=str, required=False) +# Default to WebQSP split +parser.add_argument("--num_train", type=int, default=2826) +parser.add_argument("--num_val", type=int, default=246) +parser.add_argument("--num_test", type=int, default=1628) + +args = parser.parse_args() # %% -ds = UpdatedWebQSPDataset('benchmark_archs') +hidden_channels = args.hidden_channels +lr = args.learning_rate +epochs = args.epochs +batch_size = args.batch_size +eval_batch_size = args.eval_batch_size + +# %% +if not args.custom_dataset: + ds = UpdatedWebQSPDataset('benchmark_archs') +else: + # We just assume that the size of the dataset accomodates the + # train/val/test split, because checking may be expensive. + dataset = torch.load(args.dataset_path) + + class MockDataset: + """Utility class to patch the fields in WebQSPDataset used by GRetriever.""" + def __init__(self) -> None: + pass + + @property + def split_idxs(self) -> dict: + # Imitates the WebQSP split method + return { + "train": torch.arange(args.num_train), + "val": torch.arange(args.num_val) + args.num_train, + "test": torch.arange(args.num_test) + args.num_train + args.num_val, + } + def __getitem__(self, idx: int): + return dataset[idx] + ds = MockDataset() # %% model_names = [] From a22cb82d7ff2e835a0d7b3768bac59a45fcadcad Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Sat, 24 Aug 2024 04:22:38 -0700 Subject: [PATCH 677/752] automate gretriever benchmarking for other subgraphs --- .../llm_plus_gnn/benchmark_model_archs.py | 12 ++- examples/llm_plus_gnn/rag_generate.py | 96 +++++++++++-------- 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index 651c95acdbd1..44d375e428dc 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -15,9 +15,8 @@ parser.add_argument("--epochs", type=int, default=2) parser.add_argument("--batch_size", type=int, default=8) parser.add_argument("--eval_batch_size", type=int, default=16) -parser.add_argument("--tiny_llama", store_true=True) +parser.add_argument("--tiny_llama", action='store_true') -parser.add_argument("--custom_dataset", store_true=True) parser.add_argument("--dataset_path", type=str, required=False) # Default to WebQSP split parser.add_argument("--num_train", type=int, default=2826) @@ -34,7 +33,7 @@ eval_batch_size = args.eval_batch_size # %% -if not args.custom_dataset: +if not args.dataset_path: ds = UpdatedWebQSPDataset('benchmark_archs') else: # We just assume that the size of the dataset accomodates the @@ -71,11 +70,14 @@ def __getitem__(self, idx: int): model_classes.append(GRetriever) kwargs = dict(gnn_hidden_channels=hidden_channels, num_gnn_layers=n_layer, gnn_to_use=models[m_type]) + if args.tiny_llama: + kwargs['llm_to_use'] = 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' + kwargs['mlp_out_dim'] = 2048 + kwargs['num_llm_params'] = 1 model_kwargs.append(kwargs) # %% benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, - skip_LLMs=False, tiny_llama=True, force=True) + skip_LLMs=False, tiny_llama=args.tiny_llama, force=True) -# TODO Argparse options diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 10412dfea87a..628353a364e1 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -1,54 +1,67 @@ # %% -from itertools import chain - -import pandas as pd -import torch -import tqdm from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore - -from torch_geometric.data import Data, get_features_for_triplets_groups -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet from torch_geometric.loader import RAGQueryLoader +from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd +import argparse + +# %% +parser = argparse.ArgumentParser(description="Generate new WebQSP subgraphs") +# TODO: Add more arguments +parser.add_argumetn("--use_pcst", action="store_true") +args = parser.parse_args() # %% -ds = UpdatedWebQSPDataset("full_dataset") +ds = UpdatedWebQSPDataset("dataset", limit=100) # %% -triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) +triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) # %% questions = ds.raw_dataset['question'] questions # %% -ground_truth_graphs = get_features_for_triplets_groups( - ds.indexer, (d['graph'] for d in ds.raw_dataset), - pre_transform=preprocess_triplet) -num_edges = len(ds.indexer._edges) +#ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +#num_edges = len(ds.indexer._edges) # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer( - model_name='sentence-transformers/all-roberta-large-v1').to(device) +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) # %% -fs, gs = create_remote_backend_from_triplets( - triplets=triplets, node_embedding_model=model, - node_method_to_call="encode", path="backend", - pre_transform=preprocess_triplet, node_method_kwargs={ - "batch_size": 256 - }, graph_db=NeighborSamplingRAGGraphStore, - feature_db=SentenceTransformerFeatureStore).load() +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() # %% -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, - seed_edges_kwargs={"k_edges": 10}, - sampler_kwargs={"num_neighbors": [40] * 3}) +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + q_emb = model.encode(query) + textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph["desc"] = desc + return out_graph + +def apply_retrieval_with_text(graph: Data, query: str) -> Tuple[Data, str]: + textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() + desc = (textual_nodes.to_csv(index=False) + "\n" + + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"])) + graph["desc"] = desc + return graph + +transform = apply_retrieval_with_text if args.use_pcst else apply_retrieval_via_pcst + +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 5}, seed_edges_kwargs={"k_edges": 5}, sampler_kwargs={"num_neighbors": [50]*2}, local_filter=transform) # %% # Accuracy Metrics to be added to Profiler @@ -61,36 +74,35 @@ def _eidx_helper(subg: Data, ground_truth: Data): subg_e = set(subg_eidx) gt_e = set(gt_eidx) return subg_e, gt_e - - def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): subg_e, gt_e = _eidx_helper(subg, ground_truth) total_e = set(range(num_edges)) tp = len(subg_e & gt_e) - tn = len(total_e - (subg_e | gt_e)) - return (tp + tn) / num_edges - - + tn = len(total_e-(subg_e | gt_e)) + return (tp+tn)/num_edges def check_retrieval_precision(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(subg_e) - - def check_retrieval_recall(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) - # %% retrieval_stats = {"precision": [], "recall": [], "accuracy": []} subgs = [] -for subg, gt in tqdm.tqdm( - zip((query_loader.query(q) for q in questions), ground_truth_graphs)): - retrieval_stats["precision"].append(check_retrieval_precision(subg, gt)) - retrieval_stats["recall"].append(check_retrieval_recall(subg, gt)) - retrieval_stats["accuracy"].append( - check_retrieval_accuracy(subg, gt, num_edges)) +node_len = [] +edge_len = [] +for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): + print(subg) subgs.append(subg) + node_len.append(subg['x'].shape[0]) + edge_len.append(subg['edge_attr'].shape[0]) +print(sum(node_len)/4700, sum(edge_len)/4700) + +for i, subg in enumerate(subgs): + subg['question'] = questions[i] + subg['label'] = ds[i]['label'] pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') torch.save(subgs, 'subg_results.pt') + From 117ec9e8b2eb5b63862aea2b5713c6061a0f9b63 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 26 Aug 2024 14:23:22 -0700 Subject: [PATCH 678/752] begin incorporating approx knn so that dataset can be fully generated --- examples/llm_plus_gnn/rag_feature_store.py | 124 +++++++++++++++------ 1 file changed, 87 insertions(+), 37 deletions(-) diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index 74eb73ec99eb..7e2e3fa541b4 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -11,6 +11,9 @@ from torch_geometric.nn.nlp import SentenceTransformer from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput from torch_geometric.typing import InputEdges, InputNodes +import gc +from torch_geometric.nn.pool import ApproxMIPSKNNIndex + # NOTE: Only compatible with Homogeneous graphs for now @@ -35,18 +38,10 @@ def edge_attr(self) -> Tensor: return self.get_tensor(group_name=(None, None), attr_name='edge_attr') def retrieve_seed_nodes(self, query: Any, k_nodes: int = 5) -> InputNodes: - """Retrieve the seed nodes for a given query using KNN search after - embedding the query. - - Args: - query (Any): Input to be embedded and used for KNN search. - k_nodes (int, optional): Number of nodes to search for in KNN - search. Defaults to 5. - - Returns: - InputNodes: Input object to be used in a PyG Sampler. - """ - return next(self._retrieve_seed_nodes_batch([query], k_nodes)) + result = next(self._retrieve_seed_nodes_batch([query], k_nodes)) + gc.collect() + torch.cuda.empty_cache() + return result def _retrieve_seed_nodes_batch(self, query: Iterable[Any], k_nodes: int) -> Iterator[InputNodes]: @@ -62,18 +57,10 @@ def _retrieve_seed_nodes_batch(self, query: Iterable[Any], yield indices def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: - """Retrieve the seed edges for a given query using KNN search after - embedding the query. - - Args: - query (Any): Input to be embedded and used for KNN search. - k_edges (int, optional): Number of edges to search for in KNN - search. Defaults to 3. - - Returns: - InputNodes: Input object to be used in a PyG Sampler. - """ - return next(self._retrieve_seed_edges_batch([query], k_edges)) + result = next(self._retrieve_seed_edges_batch([query], k_edges)) + gc.collect() + torch.cuda.empty_cache() + return result def _retrieve_seed_edges_batch(self, query: Iterable[Any], k_edges: int) -> Iterator[InputEdges]: @@ -93,17 +80,7 @@ def _retrieve_seed_edges_batch(self, query: Iterable[Any], def load_subgraph( self, sample: Union[SamplerOutput, HeteroSamplerOutput] ) -> Union[Data, HeteroData]: - """Retrieve subgraph features corresponding to the given sampler - output. - Args: - sample (Union[SamplerOutput, HeteroSamplerOutput]): Output from a - PyG Sampler to retrieve subgraph features for. - - Returns: - Union[Data, HeteroData]: Data object containing subgraph features - and edge indices. - """ if isinstance(sample, HeteroSamplerOutput): raise NotImplementedError @@ -119,9 +96,82 @@ def load_subgraph( node_idx=node_id, edge_idx=edge_id) +# TODO: Refactor because composition >> inheritance + +def _add_features_to_knn_index(knn_index: ApproxMIPSKNNIndex, emb: Tensor, device: torch.device, batch_size: int = 2**20): + """Add new features to the existing KNN index in batches. + + Args: + knn_index (ApproxMIPSKNNIndex): Index to add features to. + emb (Tensor): Embeddings to add. + device (torch.device): Device to store in + batch_size (int, optional): Batch size to iterate by. Defaults to 2**20, which equates to 4GB if working with 1024 dim floats. + """ + for i in range(0, emb.size(0), batch_size): + if emb.size(0) - i >= batch_size: + emb_batch = emb[i:i+batch_size].to(device) + else: + emb_batch = emb[i:].to(device) + knn_index.add(emb_batch) + +class ApproxKNNRAGFeatureStore(KNNRAGFeatureStore): + def __init__(self, enc_model: Type[Module], + model_kwargs: Optional[Dict[str, + Any]] = None, *args, **kwargs): + # TODO: Add kwargs for approx KNN to parameters here. + super().__init__(enc_model, model_kwargs, *args, **kwargs) + self.node_knn_index = None + self.edge_knn_index = None + + def _retrieve_seed_nodes_batch(self, query: Iterable[Any], + k_nodes: int) -> Iterator[InputNodes]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError + + enc_model = self.enc_model.to(self.device) + query_enc = enc_model.encode(query, + **self.model_kwargs).to(self.device) + del enc_model + gc.collect() + torch.cuda.empty_cache() + + if self.node_knn_index is None: + self.node_knn_index = ApproxMIPSKNNIndex(num_cells=100, num_cells_to_visit=100, bits_per_vector=4) + # Need to add in batches to avoid OOM + _add_features_to_knn_index(self.node_knn_index, self.x, self.device) + + output = self.node_knn_index.search(query_enc, k=k_nodes) + yield from output.index + + def _retrieve_seed_edges_batch(self, query: Iterable[Any], + k_edges: int) -> Iterator[InputEdges]: + if isinstance(self.meta, dict) and self.meta.get("is_hetero", False): + raise NotImplementedError + + enc_model = self.enc_model.to(self.device) + query_enc = enc_model.encode(query, + **self.model_kwargs).to(self.device) + del enc_model + gc.collect() + torch.cuda.empty_cache() + + if self.edge_knn_index is None: + self.edge_knn_index = ApproxMIPSKNNIndex(num_cells=100, num_cells_to_visit=100, bits_per_vector=4) + # Need to add in batches to avoid OOM + _add_features_to_knn_index(self.edge_knn_index, self.edge_attr, self.device) + + output = self.edge_knn_index.search(query_enc, k=k_edges) + yield from output.index + + + +# TODO: These two classes should be refactored class SentenceTransformerFeatureStore(KNNRAGFeatureStore): def __init__(self, *args, **kwargs): - kwargs['model_name'] = \ - kwargs.get('model_name', - 'sentence-transformers/all-roberta-large-v1') + kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') super().__init__(SentenceTransformer, *args, **kwargs) + +class SentenceTransformerApproxFeatureStore(ApproxKNNRAGFeatureStore): + def __init__(self, *args, **kwargs): + kwargs['model_name'] = kwargs.get('model_name','sentence-transformers/all-roberta-large-v1') + super().__init__(SentenceTransformer, *args, **kwargs) \ No newline at end of file From e23889c15984e569eb49e7c42d4a6fa974c74c1f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 26 Aug 2024 14:33:53 -0700 Subject: [PATCH 679/752] experimental batch query support for rag loader --- .../llm_plus_gnn/rag_generate_multihop_.py | 52 +++++++++++++++++++ torch_geometric/loader/rag_loader.py | 27 +++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 examples/llm_plus_gnn/rag_generate_multihop_.py diff --git a/examples/llm_plus_gnn/rag_generate_multihop_.py b/examples/llm_plus_gnn/rag_generate_multihop_.py new file mode 100644 index 000000000000..7b0df0dfb28c --- /dev/null +++ b/examples/llm_plus_gnn/rag_generate_multihop_.py @@ -0,0 +1,52 @@ +# %% +from profiling_utils import create_remote_backend_from_triplets +from rag_feature_store import SentenceTransformerFeatureStore +from rag_graph_store import NeighborSamplingRAGGraphStore +from torch_geometric.loader import RAGQueryLoader +from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.data import get_features_for_triplets_groups, Data +from itertools import chain +import torch +from typing import Tuple +import tqdm +import pandas as pd + + +# %% +triplets = torch.load('wikimultihopqa_full_graph.pt') + +# %% +df = pd.read_csv('wikimultihopqa_cleaned.csv') +questions = df['question_text'].to_list() + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +# %% +print("Generating remote backend...") +fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() + +# %% +print("Retrieving subgraphs...") +df_textual_nodes = pd.read_csv('wikimultihopqa_textual_nodes.csv') +df_textual_edges = pd.read_csv('wikimultihopqa_textual_edges.csv') + +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + q_emb = model.encode(query) + textual_nodes = df_textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = df_textual_edges.iloc[graph["edge_idx"]].reset_index() + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph["desc"] = desc + return out_graph + +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=apply_retrieval_via_pcst) + +# %% +subgs = [] +for subg in tqdm.tqdm(query_loader.batch_query(questions)): + print(subg) + subgs.append(subg) + +torch.save(subgs, 'subg_results.pt') diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 33d6cf0e868e..7273b34d224d 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union +from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union, Iterator from torch_geometric.data import Data, FeatureStore, HeteroData from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput @@ -86,6 +86,31 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], self.sampler_kwargs = sampler_kwargs or {} self.loader_kwargs = loader_kwargs or {} + def batch_query(self, queries: Any, batch_size: int = 512) -> Iterator[Data]: + """Retrieves subraphs associated with each query in the batch (experimental).""" + + for i in range(0, len(queries), batch_size): + if i + batch_size <= len(queries): + batch = queries[i:i + batch_size] + else: + batch = queries[i:] + + seed_nodes = self.feature_store._retrieve_seed_nodes_batch( + batch, **self.seed_nodes_kwargs) + seed_edges = self.feature_store._retrieve_seed_edges_batch( + batch, **self.seed_edges_kwargs) + + for (seed_node, seed_edge, query) in zip(seed_nodes, seed_edges, batch): + subgraph_sample = self.graph_store.sample_subgraph( + seed_node, seed_edge, **self.sampler_kwargs) + + data = self.feature_store.load_subgraph(sample=subgraph_sample, + **self.loader_kwargs) + + if self.local_filter: + data = self.local_filter(data, query) + yield data + def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature attributes. From 68a2070f71a5aa61eac40f952ec9aac54a72d284 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 26 Aug 2024 15:00:34 -0700 Subject: [PATCH 680/752] rag generate specify num samples --- examples/llm_plus_gnn/rag_generate.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 628353a364e1..046a941e70d7 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -16,12 +16,13 @@ # %% parser = argparse.ArgumentParser(description="Generate new WebQSP subgraphs") -# TODO: Add more arguments -parser.add_argumetn("--use_pcst", action="store_true") +# TODO: Add more arguments for configuring rag params +parser.add_argument("--use_pcst", action="store_true") +parser.add_argument("--num_samples", type=int, default=50) args = parser.parse_args() # %% -ds = UpdatedWebQSPDataset("dataset", limit=100) +ds = UpdatedWebQSPDataset("dataset", limit=args.num_samples) # %% triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) @@ -59,7 +60,7 @@ def apply_retrieval_with_text(graph: Data, query: str) -> Tuple[Data, str]: graph["desc"] = desc return graph -transform = apply_retrieval_with_text if args.use_pcst else apply_retrieval_via_pcst +transform = apply_retrieval_via_pcst if args.use_pcst else apply_retrieval_with_text query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 5}, seed_edges_kwargs={"k_edges": 5}, sampler_kwargs={"num_neighbors": [50]*2}, local_filter=transform) @@ -97,7 +98,7 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subgs.append(subg) node_len.append(subg['x'].shape[0]) edge_len.append(subg['edge_attr'].shape[0]) -print(sum(node_len)/4700, sum(edge_len)/4700) +print(sum(node_len)/args.num_samples, sum(edge_len)/args.num_samples) for i, subg in enumerate(subgs): subg['question'] = questions[i] From 3d5c2cb99e4f4dab12a743a3a34dfe40a8e4d5a6 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 26 Aug 2024 15:03:08 -0700 Subject: [PATCH 681/752] switch to approx knn for rag generate multihop example --- examples/llm_plus_gnn/rag_generate_multihop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/rag_generate_multihop.py index 8dfabc52647a..c4e0f6ad7544 100644 --- a/examples/llm_plus_gnn/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/rag_generate_multihop.py @@ -3,7 +3,7 @@ import torch import tqdm from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore +from rag_feature_store import SentenceTransformerApproxFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore from torch_geometric.datasets.updated_web_qsp_dataset import ( @@ -32,7 +32,7 @@ pre_transform=preprocess_triplet, node_method_kwargs={ "batch_size": 256 }, graph_db=NeighborSamplingRAGGraphStore, - feature_db=SentenceTransformerFeatureStore).load() + feature_db=SentenceTransformerApproxFeatureStore).load() # %% query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, From 9c972e77edbf84aa9d6d2ec39cd46e6c77170b6b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Mon, 26 Aug 2024 18:23:23 -0700 Subject: [PATCH 682/752] argparse for rag generate --- examples/llm_plus_gnn/benchmark_model_archs.py | 2 +- examples/llm_plus_gnn/rag_generate.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index 44d375e428dc..ddfe18950617 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -63,7 +63,7 @@ def __getitem__(self, idx: int): model_kwargs = [] model_type = ["GAT", "MLP"] models = {"GAT": GAT, "MLP": MLP} -num_layers = [1, 4, 16] +num_layers = [4] for m_type in model_type: for n_layer in num_layers: model_names.append(f"{m_type}_{n_layer}") diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 046a941e70d7..6eea08a6cdde 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -19,6 +19,7 @@ # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") parser.add_argument("--num_samples", type=int, default=50) +parser.add_argument("--out_file", default="subg_results.pt") args = parser.parse_args() # %% @@ -105,5 +106,5 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subg['label'] = ds[i]['label'] pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') -torch.save(subgs, 'subg_results.pt') +torch.save(subgs, args.out_file) From 518a9e32cbe663930b8e15e5fc9ace1195e7ed33 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Tue, 27 Aug 2024 02:51:50 -0700 Subject: [PATCH 683/752] move all benchmark code into own directory --- examples/llm_plus_gnn/__init__.py | 0 examples/llm_plus_gnn/benchmark/__init__.py | 0 .../{ => benchmark}/benchmark_model_archs.py | 5 +++-- .../{ => benchmark}/benchmark_model_gnn_enc.py | 4 ++-- examples/llm_plus_gnn/{ => benchmark}/rag_generate.py | 9 +-------- .../{ => benchmark}/rag_generate_multihop.py | 0 6 files changed, 6 insertions(+), 12 deletions(-) create mode 100644 examples/llm_plus_gnn/__init__.py create mode 100644 examples/llm_plus_gnn/benchmark/__init__.py rename examples/llm_plus_gnn/{ => benchmark}/benchmark_model_archs.py (99%) rename examples/llm_plus_gnn/{ => benchmark}/benchmark_model_gnn_enc.py (100%) rename examples/llm_plus_gnn/{ => benchmark}/rag_generate.py (92%) rename examples/llm_plus_gnn/{ => benchmark}/rag_generate_multihop.py (100%) diff --git a/examples/llm_plus_gnn/__init__.py b/examples/llm_plus_gnn/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/llm_plus_gnn/benchmark/__init__.py b/examples/llm_plus_gnn/benchmark/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark/benchmark_model_archs.py similarity index 99% rename from examples/llm_plus_gnn/benchmark_model_archs.py rename to examples/llm_plus_gnn/benchmark/benchmark_model_archs.py index ddfe18950617..9b92c74134cc 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark/benchmark_model_archs.py @@ -1,12 +1,13 @@ """Used to benchmark the performance of an untuned/fine tuned LLM against GRetriever with various architectures and layer depths.""" # %% -from g_retriever import benchmark_models, get_loss, inference_step import argparse + +import torch from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever -import torch +from g_retriever import benchmark_models, get_loss, inference_step # %% parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") diff --git a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py b/examples/llm_plus_gnn/benchmark/benchmark_model_gnn_enc.py similarity index 100% rename from examples/llm_plus_gnn/benchmark_model_gnn_enc.py rename to examples/llm_plus_gnn/benchmark/benchmark_model_gnn_enc.py index 3b65e0043ef6..60c753aeba2e 100644 --- a/examples/llm_plus_gnn/benchmark_model_gnn_enc.py +++ b/examples/llm_plus_gnn/benchmark/benchmark_model_gnn_enc.py @@ -1,9 +1,9 @@ # %% -from g_retriever import benchmark_models, get_loss, inference_step - from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever +from g_retriever import benchmark_models, get_loss, inference_step + # %% hidden_channels = 1024 num_gnn_layers = 1 diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/benchmark/rag_generate.py similarity index 92% rename from examples/llm_plus_gnn/rag_generate.py rename to examples/llm_plus_gnn/benchmark/rag_generate.py index 6eea08a6cdde..e0b5f2b546d0 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/benchmark/rag_generate.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description="Generate new WebQSP subgraphs") # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") -parser.add_argument("--num_samples", type=int, default=50) +parser.add_argument("--num_samples", type=int, default=4700) parser.add_argument("--out_file", default="subg_results.pt") args = parser.parse_args() @@ -30,11 +30,6 @@ # %% questions = ds.raw_dataset['question'] -questions - -# %% -#ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) -#num_edges = len(ds.indexer._edges) # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -95,11 +90,9 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): node_len = [] edge_len = [] for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): - print(subg) subgs.append(subg) node_len.append(subg['x'].shape[0]) edge_len.append(subg['edge_attr'].shape[0]) -print(sum(node_len)/args.num_samples, sum(edge_len)/args.num_samples) for i, subg in enumerate(subgs): subg['question'] = questions[i] diff --git a/examples/llm_plus_gnn/rag_generate_multihop.py b/examples/llm_plus_gnn/benchmark/rag_generate_multihop.py similarity index 100% rename from examples/llm_plus_gnn/rag_generate_multihop.py rename to examples/llm_plus_gnn/benchmark/rag_generate_multihop.py From 59564a57818a2947cbb8d0bded4ae09deed449af Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 14:00:53 -0700 Subject: [PATCH 684/752] beginning of refactor into doc --- docs/source/advanced/rag.rst | 2813 ++++++++++++++++++++++++++++++++++ 1 file changed, 2813 insertions(+) create mode 100644 docs/source/advanced/rag.rst diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst new file mode 100644 index 000000000000..9400f6de9514 --- /dev/null +++ b/docs/source/advanced/rag.rst @@ -0,0 +1,2813 @@ +Working with LLM RAG in Pytorch Geometric +========================================= + +This series aims to provide a starting point and for +multi-step LLM Retrieval Augmented Generation +(RAG) using Graph Neural Networks. + +Motivation +---------- + +As Large Language Models (LLMs) quickly grow to dominate industry, they +are increasingly being deployed at scale in use cases that require very +specific contextual expertise. LLMs often struggle with these cases out +of the box, as they will hallucinate answers that are not included in +their training data. At the same time, many business already have large +graph databases full of important context that can provide important +domain-specific context to reduce hallucination and improve answer +fidelity for LLMs. Graph Neural Networks (GNNs) provide a means for +efficiently encoding this contextual information into the model, which +can help LLMs to better understand and generate answers. Hence, theres +an open research question as to how to effectively use GNN encodings +efficiently for this purpose, that the tooling provided here can help +investigate. + +Architecture +------------ + +To model the use-case of RAG from a large knowledge graph of millions of +nodes, we present the following architecture: + + + + + +.. image:: _Introduction_files/_Introduction_7_0.svg + + + +Graph RAG as shown in the diagram above follows the following order of +operations: + +0. To start, not pictured here, there must exist a large knowledge graph + that exists as a source of truth. The nodes and edges of this + knowledge graph + +During inference time, RAG implementations that follow this architecture +are composed of the following steps: + +1. Tokenize and encode the query using the LLM Encoder +2. Retrieve a subgraph of the larger knowledge graph (KG) relevant to + the query and encode it using a GNN +3. Jointly embed the GNN embedding with the LLM embedding +4. Utilize LLM Decoder to decode joint embedding and generate a response + + + + +Encoding a Large Knowledge Graph +================================ + +In this notebook, we are going to walk through how to encode a large +knowledge graph for the purposes of Graph RAG. We will provide two +examples of how to do so, along with demonstration code. + +Example 1: Building from Already Existing Datasets +-------------------------------------------------- + +In most RAG scenarios, the subset of the information corpus that gets +retrieved is crucial for whether the appropriate response to the LLM. +The same is true for GNN based RAG. Consider the following dataset +WebQSP: + +.. code:: ipython3 + + from torch_geometric.datasets import UpdatedWebQSPDataset + + num_questions = 100 + ds = UpdatedWebQSPDataset('small_sample', limit=num_questions) + + +.. parsed-literal:: + + /home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html + from .autonotebook import tqdm as notebook_tqdm + + +WebQSP is a dataset that is based off of a subset of the Freebase +Knowledge Graph, which is an open-source knowledge graph formerly +maintained by Google. For each question-answer pair in the dataset, a +subgraph was chosen based on a Semantic SPARQL search on the larger +knowledge graph, to provide relevent context on finding the answer. So +each entry in the dataset consists of: - A question to be answered - The +answer - A knowledge graph subgraph of Freebase that has the context +needed to answer the question. + +.. code:: ipython3 + + ds.raw_dataset + + + + +.. parsed-literal:: + + Dataset({ + features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'], + num_rows: 100 + }) + + + +.. code:: ipython3 + + ds.raw_dataset[0] + + + + +.. parsed-literal:: + + {'id': 'WebQTrn-0', + 'question': 'what is the name of justin bieber brother', + 'answer': ['Jaxon Bieber'], + 'q_entity': ['Justin Bieber'], + 'a_entity': ['Jaxon Bieber'], + 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'], + ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'], + ['Rudolph Valentino', + 'freebase.valuenotation.is_reviewed', + 'Place of birth'], + ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'], + ['Stephen Melton', 'people.person.nationality', 'United States of America'], + ['Record producer', + 'music.performance_role.regular_performances', + 'm.012m1vf1'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'], + ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'], + ['2011 Teen Choice Awards', + 'award.award_ceremony.awards_presented', + 'm.0yrkr34'], + ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'], + ['As Long As You Love Me (Ferry Corsten radio)', + 'common.topic.notable_types', + 'Musical Recording'], + ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'], + ['Stratford', 'location.location.containedby', 'Canada'], + ['Singer', + 'base.lightweight.profession.specialization_of', + 'Musicians and Singers'], + ['Enrique Iglesias', 'people.person.profession', 'Singer'], + ['Beauty and a Beat (acoustic version)', + 'music.recording.artist', + 'Justin Bieber'], + ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'], + ['50 Cent', 'people.person.profession', 'Film Producer'], + ['As Long As You Love Me (Audien dubstep mix)', + 'music.recording.canonical_version', + 'As Long As You Love Me'], + ['Kevin Risto', 'people.person.gender', 'Male'], + ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'], + ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'], + ['Mary J. Blige', 'people.person.profession', 'Record producer'], + ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'], + ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'], + ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'], + ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'], + ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'], + ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'], + ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'], + ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'], + ['American Music Award for Favorite Pop/Rock Album', + 'award.award_category.winners', + 'm.0ndc259'], + ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'], + ['Ontario', 'location.administrative_division.country', 'Canada'], + ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'], + ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'], + ['Billboard Music Award for Top Streaming Artist', + 'award.award_category.winners', + 'm.0njhx1b'], + ['Justin Bieber', 'film.producer.film', "Justin Bieber's Believe"], + ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'], + ['Brandy Norwood', 'people.person.profession', 'Singer'], + ["Justin Bieber's Believe", 'film.film.personal_appearances', 'm.0101ft2j'], + ['Justin Bieber', 'music.artist.album', 'All Bad'], + ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'], + ['m.0v_729v', + 'tv.tv_guest_personal_appearance.episode', + 'Results Show: Week 7'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'], + ['One Less Lonely Girl', + 'music.album.primary_release', + 'One Less Lonely Girl'], + ['Twista', 'people.person.gender', 'Male'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'], + ['Ciara', 'broadcast.artist.content', 'FLOW 103'], + ['Jon M. Chu', 'film.director.film', "Justin Bieber's Believe"], + ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'], + ['Toby Gad', 'music.artist.genre', 'Rock music'], + ['Madonna', 'music.artist.genre', 'Pop music'], + ['Selena Gomez', 'music.artist.genre', 'Europop'], + ['m.0gbm3cg', + 'film.personal_film_appearance.film', + 'Justin Bieber: Never Say Never'], + ['Baby', 'music.recording.canonical_version', 'Baby'], + ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'], + ['Boyfriend', 'music.recording.artist', 'Justin Bieber'], + ['Dr. Dre', 'music.artist.genre', 'Rap music'], + ['MTV Video Music Award Japan for Best New Artist', + 'award.award_category.winners', + 'm.0yrhrwc'], + ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'], + ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'], + ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'], + ['m.0gctwjk', + 'tv.tv_guest_role.episodes_appeared_in', + 'Series 2, Episode 3'], + ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'], + ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'], + ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'], + ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'], + ['Selena Gomez', 'people.person.profession', 'Dancer'], + ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'], + ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'], + ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'], + ['As Long As You Love Me (PAULO & JACKINSKY radio)', + 'common.topic.notable_types', + 'Musical Recording'], + ['Beauty and a Beat', + 'music.single.versions', + 'Beauty and a Beat (Wideboys Club Mix)'], + ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Bryan Adams', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Madonna', 'people.person.profession', 'Singer-songwriter'], + ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'], + ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'], + ['Terence Dudley', 'music.artist.genre', 'Reggae'], + ['Kylie Minogue', 'people.person.profession', 'Actor'], + ['Adrienne Bailon', 'music.artist.genre', 'Pop music'], + ['Katy Perry', 'music.artist.genre', 'Electronic music'], + ['Dany Brillant', 'people.person.gender', 'Male'], + ['Martin Kierszenbaum', 'people.person.gender', 'Male'], + ['Anastacia', 'people.person.nationality', 'United States of America'], + ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'], + ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'], + ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Somebody to Love', 'music.composition.form', 'Song'], + ['Teen Choice Award for Choice Twitter Personality', + 'award.award_category.winners', + 'm.0yrkr34'], + ['Chef Tone', 'people.person.place_of_birth', 'Chicago'], + ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'], + ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'], + ['Record producer', + 'fictional_universe.character_occupation.characters_with_this_occupation', + 'Haley James Scott'], + ['Colbie Caillat', 'music.artist.genre', 'Pop music'], + ['C1', 'music.artist.genre', 'Contemporary R&B'], + ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'], + ['Kanye West', 'people.person.profession', 'Singer'], + ['Pop music', 'common.topic.subject_of', 'Stephen Melton'], + ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'], + ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'], + ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'], + ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'], + ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'], + ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'], + ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'], + ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'], + ['Live My Life', 'music.album.compositions', 'Live My Life'], + ['RedOne', 'music.artist.genre', 'Rock music'], + ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'], + ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'], + ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'], + ['Little Bird', 'common.topic.notable_types', 'Musical Recording'], + ['As Long As You Love Me (Ferry Corsten radio)', + 'music.recording.featured_artists', + 'Big Sean'], + ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'], + ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'], + ['Terius Nash', 'people.person.nationality', 'United States of America'], + ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'], + ['Athan Grace', 'people.person.profession', 'Actor'], + ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'], + ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'], + ['All Around the World (acoustic version)', + 'music.recording.artist', + 'Justin Bieber'], + ['Bad Day', 'music.composition.composer', 'Marvin Isley'], + ['Brandy Norwood', + 'influence.influence_node.influenced_by', + 'Whitney Houston'], + ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['MTV Video Music Award for Artist to Watch', + 'award.award_category.winners', + 'm.0n1ykxp'], + ['Caitlin Beadles', + 'celebrities.celebrity.sexual_relationships', + 'm.0d33gyj'], + ['As Long As You Love Me', + 'music.single.versions', + 'As Long As You Love Me (Audiobot instrumental)'], + ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'], + ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'], + ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'The Mighty Mighty Bosstones'], + ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'], + ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'], + ['Lolly', 'music.composition.composer', 'Juicy J'], + ['Justin Bieber: Never Say Never', + 'media_common.netflix_title.netflix_genres', + 'Documentary film'], + ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'], + ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Amerie', 'music.artist.genre', 'Pop music'], + ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'], + ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'], + ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'], + ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'], + ['As Long As You Love Me', + 'music.single.versions', + 'As Long As You Love Me (Audien dubstep edit)'], + ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'], + ['Recovery', 'music.recording.song', 'Recovery'], + ['Justin Timberlake', 'music.artist.genre', 'Electronic music'], + ['Mannie Fresh', 'people.person.nationality', 'United States of America'], + ['m.0101ftqp', 'film.film_crew_gig.film', "Justin Bieber's Believe"], + ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'], + ['Leif Garrett', 'music.artist.genre', 'Rock music'], + ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'], + ['First Dance', 'music.recording.artist', 'Justin Bieber'], + ['#thatPower', 'music.recording.song', '#thatPower'], + ['Children', 'rdf-schema#range', 'Person'], + ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'], + ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'], + ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'], + ['2013 Teen Choice Awards', + 'award.award_ceremony.awards_presented', + 'm.0wjgqck'], + ['The Island Def Jam Music Group', + 'organization.organization.parent', + 'm.04q65lb'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'Rusted Root'], + ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'], + ['m.0z87d3n', + 'award.award_honor.award', + 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'], + ['Shaffer Smith', 'music.artist.genre', 'Dance music'], + ['Live My Life', 'music.composition.composer', 'John Mamann'], + ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'], + ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'], + ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], + ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'], + ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'], + ['Tampa', 'location.location.containedby', 'United States of America'], + ['Love Never Felt So Good', + 'music.album.compositions', + 'Love Never Felt So Good'], + ['As Long As You Love Me (Ferry Corsten remix)', + 'music.recording.artist', + 'Justin Bieber'], + ['Nelly', 'music.artist.genre', 'Rhythm and blues'], + ['Marvin Isley', 'music.composer.compositions', 'Bad Day'], + ['Somebody to Love', 'common.topic.notable_types', 'Composition'], + ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'], + ['Snoop Dogg', 'people.person.gender', 'Male'], + ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'], + ['Estelle', 'people.person.profession', 'Record producer'], + ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'], + ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'], + ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'], + ['50 Cent', 'people.person.nationality', 'United States of America'], + ['Chris Jasper', 'people.person.gender', 'Male'], + ['Sir Nolan', 'music.artist.genre', 'Pop music'], + ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'], + ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'], + ['Snoop Dogg', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['David Nicksay', 'people.person.gender', 'Male'], + ['Justin Bieber', 'people.person.profession', 'Record producer'], + ['Everlast', 'people.person.profession', 'Singer-songwriter'], + ['Juno Awards of 2014', + 'award.award_ceremony.awards_presented', + 'm.0102z0vx'], + ['As Long As You Love Me (Audiobot remix)', + 'music.recording.song', + 'As Long as You Love Me'], + ['#thatPower', 'music.composition.composer', 'Will i Am'], + ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'], + ['m.0_cyzs_', + 'celebrities.legal_entanglement.offense', + 'Driving under the influence'], + ['LeAnn Rimes', 'people.person.profession', 'Actor'], + ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'], + ['Mann', 'people.person.gender', 'Male'], + ['JoJo', 'people.person.gender', 'Female'], + ['Right Here (featuring Drake)', + 'music.recording.canonical_version', + 'Right Here'], + ['Mason Levy', 'music.composer.compositions', 'Boyfriend'], + ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'], + ['m.0yrjynf', + 'award.award_honor.award', + 'Teen Choice Award for Choice Summer Music Star: Male'], + ['Pras', 'people.person.profession', 'Record producer'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'], + ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'], + ['My World 2.0', 'music.album.releases', 'My World 2.0'], + ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'], + ['Marvin Isley', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'], + ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'], + ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'], + ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'], + ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'], + ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['School Gyrls', 'film.film.language', 'English Language'], + ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'], + ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'], + ['Katy Perry', 'people.person.profession', 'Actor'], + ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'], + ['Ronald Isley', 'people.person.profession', 'Actor'], + ['Live My Life (Party Rock remix)', + 'music.recording.featured_artists', + 'Redfoo'], + ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'], + ['Jaxon Bieber', 'people.person.nationality', 'Canada'], + ['As Long as You Love Me (album version)', + 'common.topic.notable_types', + 'Musical Recording'], + ['Justin Bieber: Just Getting Started', + 'book.written_work.author', + 'Justin Bieber'], + ['BeirutNights.com Radio', + 'broadcast.content.artist', + 'Marc Maris vs. Ramone'], + ['Gwen Stefani', 'people.person.profession', 'Musician'], + ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'], + ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'], + ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'], + ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'], + ['Love Never Felt So Good', + 'music.album.releases', + 'Love Never Felt So Good'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'], + ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'], + ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', + 'music.recording.canonical_version', + 'Beauty and a Beat'], + ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'], + ['Dance music', + 'broadcast.genre.content', + "PartyRadioUSA.net - Nation's #1 Party Authority"], + ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'], + ['Terius Nash', 'people.person.profession', 'Record producer'], + ['Terence Dudley', 'people.person.profession', 'Record producer'], + ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'], + ['Baby', 'common.topic.notable_types', 'Award-Winning Work'], + ['Lolly', 'music.recording.canonical_version', 'Lolly'], + ['Scooter Braun', 'people.person.gender', 'Male'], + ['Mistletoe', 'music.album.artist', 'Justin Bieber'], + ['Sir Nolan', 'people.person.gender', 'Male'], + ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'], + ["Justin Bieber's Believe", 'film.film.other_crew', 'm.0101ftt1'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'], + ['Synthpop', 'music.genre.parent_genre', 'K-pop'], + ['Adam Messinger', + 'music.composer.compositions', + "Turn to You (Mother's Day Dedication)"], + ['m.0yrktlv', + 'award.award_honor.award', + 'Teen Choice Award for Choice Male Hottie'], + ['Kanye West', 'people.person.nationality', 'United States of America'], + ['Iggy Azalea', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv4c'], + ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'], + ['JellyRadio.com', 'broadcast.content.artist', 'DMX'], + ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'], + ['m.0gxnnzy', + 'celebrities.romantic_relationship.relationship_type', + 'Dated'], + ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'], + ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'], + ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'], + ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fvcp'], + ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'], + ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Britney Spears', + 'broadcast.artist.content', + "PartyRadioUSA.net - Nation's #1 Party Authority"], + ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'], + ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'], + ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'], + ['Ludacris', 'music.artist.contribution', 'm.0vp800w'], + ['Singer', 'common.topic.subject_of', 'Justin Bieber'], + ['Fergie', 'music.artist.genre', 'Rock music'], + ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'], + ['Toby Gad', 'people.person.profession', 'Record producer'], + ['All Around The World', 'music.composition.composer', 'Justin Bieber'], + ['Mistletoe', 'music.album.release_type', 'Single'], + ['Kid Cudi', 'people.person.profession', 'Film Producer'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'], + ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'], + ['Live My Life (Party Rock remix)', + 'music.recording.tracks', + 'Live My Life (Party Rock remix)'], + ['Beauty and a Beat (Bisbetic Instrumental)', + 'music.recording.artist', + 'Justin Bieber'], + ['m.0njw4z2', + 'award.award_honor.award', + 'MTV Europe Music Award for Best Male'], + ["Destiny's Child", 'music.artist.genre', 'Contemporary R&B'], + ['Snoop Dogg', 'people.person.profession', 'Record producer'], + ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'], + ['m.0gbm3c3', + 'film.personal_film_appearance.type_of_appearance', + 'Him/Herself'], + ['Rodney Jerkins', 'people.person.nationality', 'United States of America'], + ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'], + ["PartyRadioUSA.net - Nation's #1 Party Authority", + 'broadcast.content.artist', + 'Miley Cyrus'], + ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'], + ["Destiny's Child", 'music.artist.genre', 'Pop music'], + ['United States of America', + 'base.biblioness.bibs_topic.is_really', + 'United States of America'], + ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['m.09xx941', 'common.webpage.topic', 'Teen idol'], + ['Christina Milian', 'people.person.profession', 'Record producer'], + ['JoJo', 'people.person.nationality', 'United States of America'], + ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'], + ['Next to You', 'music.album.release_type', 'Single'], + ['#thatPower', 'music.composition.recordings', '#thatPOWER'], + ['Willa Ford', 'people.person.languages', 'English Language'], + ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['All That Matters', 'music.composition.composer', 'Andre Harris'], + ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'], + ['Paul Anka', 'music.artist.genre', 'Pop music'], + ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], + ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'], + ['Caitlin Beadles', 'people.person.nationality', 'Canada'], + ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'], + ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'], + ['Hot Wired Radio', + 'broadcast.content.broadcast', + 'Hot Wired Radio - 128kbps Stream'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'], + ['Avery', 'common.topic.notable_types', 'Musical Artist'], + ['m.0gbm3d9', + 'film.personal_film_appearance.film', + 'Justin Bieber: Never Say Never'], + ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Beyoncé Knowles', 'people.person.profession', 'Actor'], + ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Tupac Shakur', 'people.person.profession', 'Actor'], + ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'], + ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'], + ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'], + ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Frank Ocean', 'people.person.nationality', 'United States of America'], + ['Christina Milian', 'music.composer.compositions', 'Baby'], + ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'], + ['Justin Timberlake', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Khalil', 'people.person.gender', 'Male'], + ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'], + ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'], + ['Selena Gomez', + 'freebase.valuenotation.has_no_value', + 'Spouse (or domestic partner)'], + ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], + ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'], + ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'], + ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv7x'], + ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'], + ['Anastacia', 'music.artist.genre', 'Contemporary R&B'], + ['C1', 'music.artist.genre', 'Hip hop music'], + ['My Worlds Acoustic', + 'freebase.valuenotation.is_reviewed', + 'Album content type'], + ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'], + ['Live My Life', 'music.composition.language', 'English Language'], + ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'], + ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'], + ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'], + ['Baby', 'music.album.releases', 'Baby'], + ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'], + ['Right Here', 'music.recording.canonical_version', 'Right Here'], + ['Justin Bieber', 'people.person.profession', 'Musician'], + ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Bigger', 'music.composition.composer', 'Waynne Nugent'], + ['Home to Mama', 'music.composition.composer', 'Cody Simpson'], + ['Big R Radio - The Hawk', + 'broadcast.content.artist', + 'The Black Eyed Peas'], + ['Thought Of You', 'music.composition.composer', 'Justin Bieber'], + ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'], + ['Singer', 'people.profession.specializations', 'Prima donna'], + ['Alanis Morissette', 'people.person.profession', 'Record producer'], + ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'], + ['Record producer', 'common.topic.notable_for', 'g.1258k9617'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'], + ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'], + ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'], + ['Justin Bieber: Never Say Never', + 'film.film.production_companies', + 'AEG Live'], + ['Redfoo', 'people.person.gender', 'Male'], + ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'], + ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'], + ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'], + ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'], + ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'], + ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'], + ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Right Here', 'music.album.featured_artists', 'Drake'], + ['m.01053qzf', + 'film.personal_film_appearance.film', + 'Justin Bieber: Never Say Never'], + ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'], + ['Jordan Pruitt', 'music.artist.genre', 'Pop music'], + ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'], + ['Thought of You', 'common.topic.notable_types', 'Canonical Version'], + ['Whitney Houston', 'people.person.profession', 'Record producer'], + ['m.07lkzw7', 'common.webpage.category', 'Official Website'], + ['Ray J', 'people.person.profession', 'Musician'], + ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'], + ['Enrique Iglesias', 'people.person.gender', 'Male'], + ['m.0101fv5f', + 'film.film_regional_release_date.film', + "Justin Bieber's Believe"], + ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'], + ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'], + ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'], + ['Selena Gomez', 'music.artist.genre', 'Teen pop'], + ["Justin Bieber's Believe", 'film.film.produced_by', 'Scooter Braun'], + ['Love Never Felt So Good', 'music.album.genre', 'Disco'], + ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'], + ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'], + ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['m.0v_729v', + 'tv.tv_guest_personal_appearance.appearance_type', + 'Guest host'], + ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'], + ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'], + ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'], + ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'], + ['All Around the World', 'music.recording.featured_artists', 'Ludacris'], + ['Christina Milian', 'people.person.profession', 'Actor'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'], + ['Dance music', 'broadcast.genre.content', '181-party'], + ['Queen Elizabeth II Diamond Jubilee Medal', + 'award.award_category.winners', + 'm.0njwqrb'], + ['Sean Kingston', 'people.person.profession', 'Singer'], + ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'], + ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'], + ['CMT Music Award: Collaborative Video of the Year', + 'award.award_category.winners', + 'm.0njvs9s'], + ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'], + ['One Time', 'common.topic.notable_types', 'Musical Album'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'], + ['Katy Perry', 'music.artist.genre', 'Disco'], + ['Chingy', 'people.person.profession', 'Actor'], + ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'], + ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'], + ['Rihanna', 'music.artist.genre', 'Dance-pop'], + ['Justin Bieber', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'], + ['Chingy', 'people.person.gender', 'Male'], + ['Reed Smoot', 'people.person.gender', 'Male'], + ["Justin Bieber's Believe", 'film.film.edited_by', 'Jillian Twigger Moul'], + ['Teyana', 'freebase.valuenotation.has_value', 'Parents'], + ['Next to You', 'music.recording.song', 'Next to You'], + ['All Bad', 'music.composition.composer', 'Jason \\"Poo Bear\\" Boyd'], + ['As Long as You Love Me', + 'music.album.releases', + 'As Long As You Love Me (remixes)'], + ['Teen Choice Award for Choice Music: Breakout Artist - Male', + 'award.award_category.winners', + 'm.0yrjvlh'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'], + ['Singer', 'common.topic.article', 'm.09l6h'], + ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'], + ['Scooter Braun', 'film.producer.film', "Justin Bieber's Believe"], + ['Justin Bieber: Never Say Never', + 'award.award_winning_work.awards_won', + 'm.0pc670l'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'], + ['Beauty And A Beat', 'music.composition.form', 'Song'], + ['Britney Spears', 'music.artist.genre', 'Electronic dance music'], + ['HitzRadio.com', 'broadcast.content.artist', "Destiny's Child"], + ['Beyoncé Knowles', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Live My Life', 'music.recording.tracks', 'Live My Life'], + ['m.0njhyh_', + 'award.award_honor.award', + 'Billboard Music Award for Top Streaming Song (Video)'], + ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'], + ['Ludacris', 'people.person.nationality', 'United States of America'], + ['Justin Bieber: Never Say Never', + 'film.film.film_production_design_by', + 'Devorah Herbert'], + ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'], + ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'], + ['Drake', 'music.artist.genre', 'Rhythm and blues'], + ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'], + ['Nick Jonas', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'], + ['Lupe Fiasco', + 'broadcast.artist.content', + "PartyRadioUSA.net - Nation's #1 Party Authority"], + ['Martin Kierszenbaum', + 'people.person.place_of_birth', + 'United States of America'], + ['As Long as You Love Me', + 'music.composition.recordings', + 'As Long as You Love Me'], + ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'], + ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'], + ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'], + ['Beautiful', 'music.composition.lyricist', 'Toby Gad'], + ['Redfoo', 'music.artist.genre', 'Electronic dance music'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'], + ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'], + ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'], + ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'], + ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'], + ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'], + ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'], + ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], + ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'], + ['m.0y5tl39', + 'film.personal_film_appearance.film', + 'Les Coulisses des Golden Globes'], + ['Teen idol', 'common.topic.webpage', 'm.09y89l2'], + ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Kevin Risto', 'people.person.profession', 'Musician'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'], + ['justinbieber', + 'award.award_nominated_work.award_nominations', + 'm.0z0tgz6'], + ['Justin Bieber', 'music.artist.label', 'Island Records'], + ['Ernie Isley', 'people.person.nationality', 'United States of America'], + ['Kylie Minogue', 'people.person.profession', 'Film Producer'], + ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'], + ['Everlast', 'music.artist.label', 'Island Records'], + ['5th Annual Shorty Awards', + 'award.award_ceremony.awards_presented', + 'm.0ywvh8k'], + ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'], + ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'], + ['Baby', 'common.topic.notable_types', 'Composition'], + ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'], + ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'], + ['m.0tkqqgg', + 'award.award_nomination.award', + 'Juno Award for Pop Album of the Year'], + ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'], + ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'], + ['Person', 'type.type.properties', 'Parents'], + ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Nasri', 'people.person.profession', 'Singer'], + ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'], + ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'], + ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'], + ['As Long as You Love Me', + 'music.album.compositions', + 'As Long as You Love Me'], + ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'], + ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'], + ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'], + ['Bigger', 'music.composition.composer', 'Frank Ocean'], + ['Bigger', 'music.composition.recordings', 'Bigger'], + ['Canadian', 'common.topic.notable_types', 'Ethnicity'], + ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'], + ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'], + ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Chef Tone', 'people.person.nationality', 'United States of America'], + ['Whitney Houston', 'music.artist.genre', 'Dance music'], + ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'], + ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'], + ['Change Me', 'music.album.primary_release', 'Change Me'], + ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'], + ['m.0w3gbtv', + 'film.personal_film_appearance.film', + 'Zendaya: Behind the Scenes'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'], + ['That Should Be Me', 'music.composition.form', 'Song'], + ['Never Say Never', 'music.album.compositions', 'Never Say Never'], + ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'Justin Bieber'], + ['#thatPOWER', 'music.album.releases', '#thatPOWER'], + ['Ashley Tisdale', 'people.person.profession', 'Actor'], + ['Sir Nolan', 'music.artist.genre', 'Rock music'], + ['Beauty and a Beat (acoustic version)', + 'music.recording.song', + 'Beauty And A Beat'], + ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'], + ['Sia Furler', 'people.person.profession', 'Singer-songwriter'], + ['Usher', 'music.composer.compositions', 'First Dance'], + ['m.0n1ykxp', + 'award.award_honor.award', + 'MTV Video Music Award for Artist to Watch'], + ['Justin Bieber: Never Say Never', + 'media_common.netflix_title.netflix_genres', + 'Rockumentary'], + ['Amerie', 'people.person.gender', 'Female'], + ['Real Change: Artists for Education', + 'film.film.personal_appearances', + 'm.0y5th3r'], + ['Mistletoe', 'music.album.primary_release', 'Mistletoe'], + ['Beautiful and the Beat', + 'music.recording.canonical_version', + 'Beauty and a Beat'], + ['#Thatpower', 'music.recording.tracks', '#thatPOWER'], + ['Baby', 'common.topic.notable_types', 'Musical Album'], + ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'], + ['PYD', 'common.topic.notable_types', 'Composition'], + ['Ashlee Simpson', 'people.person.profession', 'Singer'], + ['Pray', 'music.album.artist', 'Justin Bieber'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'], + ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'], + ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'], + ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'], + ['Geri Halliwell', 'people.person.profession', 'Model'], + ['iJustine', 'people.person.gender', 'Female'], + ['Nelly Furtado', 'people.person.gender', 'Female'], + ['Trey Songz', 'people.person.nationality', 'United States of America'], + ['m.0ng_vkd', + 'film.personal_film_appearance.film', + 'A Michael Bublé Christmas'], + ["Justin Bieber's Believe", 'film.film.produced_by', "Bill O'Dowd"], + ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'], + ['Ludacris', 'music.composer.compositions', 'Baby'], + ['Terius Nash', 'music.featured_artist.recordings', 'Baby'], + ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'], + ['Vanessa Hudgens', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'], + ['Beyoncé Knowles', 'people.person.profession', 'Record producer'], + ['#thatPOWER', 'music.recording.tracks', '#thatPower'], + ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'], + ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'], + ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'], + ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'], + ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'], + ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Madonna', 'people.person.profession', 'Record producer'], + ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'], + ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'], + ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'], + ['As Long as You Love Me (acoustic version)', + 'music.recording.song', + 'As Long as You Love Me'], + ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'], + ['Blu Cantrell', 'people.person.gender', 'Female'], + ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'], + ['Rob Thomas', 'people.person.gender', 'Male'], + ['Singer', 'people.profession.specializations', 'Piano Singer'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'], + ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'], + ['m.0hvlt03', + 'film.film_film_distributor_relationship.film', + 'Justin Bieber: Never Say Never'], + ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fvq6'], + ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'], + ['justinbieber', + 'award.award_nominated_work.award_nominations', + 'm.0_srv2b'], + ['Terence Dudley', 'people.person.profession', 'Musician'], + ['Donna Summer', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['m.0101fszs', + 'film.personal_film_appearance.film', + "Justin Bieber's Believe"], + ['Alanis Morissette', + 'freebase.valuenotation.is_reviewed', + 'Official website'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'Jenna Andrews'], + ['FLOW 103', 'broadcast.content.artist', 'Cherish'], + ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Next to You', 'music.recording.song', 'Next to You'], + ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'], + ['Ray J', 'people.person.nationality', 'United States of America'], + ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'], + ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'], + ['m.0w3gbtv', + 'film.personal_film_appearance.type_of_appearance', + 'Him/Herself'], + ['Montell Jordan', 'music.artist.genre', 'Hip hop music'], + ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Fabolous', 'broadcast.artist.content', 'PowerHitz'], + ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'], + ['Nelly Furtado', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'], + ['Record producer', 'common.topic.webpage', 'm.09ygb05'], + ['As Long As You Love Me (Ferry Corsten remix)', + 'music.recording.canonical_version', + 'As Long As You Love Me'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'], + ['m.0gbm3fj', + 'film.personal_film_appearance.type_of_appearance', + 'Him/Herself'], + ['Bryan-Michael Cox', + 'freebase.valuenotation.is_reviewed', + 'Place of birth'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'], + ['As Long As You Love Me', + 'music.single.versions', + 'As Long As You Love Me (Ferry Corsten club dub)'], + ['Iggy Azalea', 'music.artist.genre', 'Synthpop'], + ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'], + ['As Long As You Love Me (Ferry Corsten club dub)', + 'common.topic.notable_types', + 'Musical Recording'], + ['#thatPOWER', 'music.album.album_content_type', 'Studio album'], + ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Katy Perry', 'music.artist.genre', 'Electronic dance music'], + ['Kid Cudi', 'people.person.profession', 'Record producer'], + ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'], + ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'], + ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Jaden Smith', 'people.person.profession', 'Dancer'], + ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'], + ['Keyshia Cole', 'people.person.profession', 'Record producer'], + ['Guest host', + 'tv.non_character_role.tv_guest_personal_appearances', + 'm.0v_98y7'], + ['Person', 'type.type.properties', 'Spouse (or domestic partner)'], + ['Fall Out Boy', 'music.artist.origin', 'Chicago'], + ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'], + ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'], + ['FLOW 103', 'broadcast.content.artist', '50 Cent'], + ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'], + ['L.A. Reid', 'music.producer.releases_produced', 'My World'], + ['L.A. Reid', 'people.person.gender', 'Male'], + ['Jessie J', 'music.artist.genre', 'Hip hop music'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'], + ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'], + ['Beauty and a Beat (Bisbetic Radio Mix)', + 'music.recording.artist', + 'Justin Bieber'], + ['London', 'location.location.containedby', 'Ontario'], + ['Justin Bieber: Never Say Never', + 'film.film.film_set_decoration_by', + 'Lia Roldan'], + ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Chris Brown', 'music.composer.compositions', 'Next to You'], + ['Beautiful', 'music.recording.tracks', 'Beautiful'], + ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'], + ['Children', 'type.property.schema', 'Person'], + ['Change Me', 'music.album.releases', 'Change Me'], + ['RedOne', 'music.artist.label', 'Island Records'], + ['School Gyrls', 'film.film.starring', 'm.0jztshx'], + ['All Around the World', + 'music.recording.canonical_version', + 'All Around the World'], + ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'], + ['Teen Choice Award for Choice Twitter Personality', + 'award.award_category.winners', + 'm.0wjhc6c'], + ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'], + ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'], + ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'], + ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'], + ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'], + ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'], + ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'], + ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'], + ["PartyRadioUSA.net - Nation's #1 Party Authority", + 'broadcast.content.artist', + 'Lupe Fiasco'], + ['Hikaru Utada', 'music.artist.label', 'Island Records'], + ['Dr. Dre', 'people.person.profession', 'Record producer'], + ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'], + ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Justin Bieber', 'music.composer.compositions', 'Change Me'], + ['Right Here', 'common.topic.notable_types', 'Composition'], + ['Change Me', 'music.composition.composer', 'Jason \\"Poo Bear\\" Boyd'], + ['Beauty and a Beat (Wideboys Radio Mix)', + 'music.recording.canonical_version', + 'Beauty and a Beat'], + ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'], + ['#Thatpower', 'music.recording.artist', 'Will i Am'], + ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'], + ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'], + ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'], + ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'], + ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'], + ['m.0v90skf', + 'award.award_honor.award', + 'Billboard Music Award for Top Male Artist'], + ['Ludacris', 'people.person.profession', 'Actor'], + ['Heartbreaker', 'music.album.genre', 'Pop music'], + ['Cameo appearance', + 'tv.special_tv_performance_type.episode_performances', + 'm.0v1lwt2'], + ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'], + ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'], + ['Model', 'base.lightweight.profession.similar_professions', 'Actor'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'], + ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'], + ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'], + ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'One Chance'], + ['Never Let You Go', 'common.topic.notable_types', 'Composition'], + ['Live My Life', 'common.topic.article', 'm.0j4453y'], + ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'], + ["Justin Bieber's Believe", 'film.film.personal_appearances', 'm.0y5t8gm'], + ['Roller Coaster', + 'award.award_nominated_work.award_nominations', + 'm.0_x4zg3'], + ['Chris Brown', 'people.person.nationality', 'United States of America'], + ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'], + ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'], + ['Teen pop', 'common.topic.article', 'm.02ny8z'], + ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'], + ['Iggy Azalea', 'people.person.gender', 'Female'], + ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Adrienne Bailon', 'people.person.profession', 'Dancer'], + ['Hip hop music', 'broadcast.genre.content', '181-beat'], + ['m.0sgk_cw', + 'award.award_honor.award', + "Kids' Choice Award for Favorite Song"], + ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'], + ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'], + ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'], + ['MTV Video Music Brazil Award for Best International Artist', + 'award.award_category.winners', + 'm.0yrhhqv'], + ['Mariah Carey', 'music.artist.label', 'Island Records'], + ['Music', 'common.topic.subject_of', 'POPPMusic.net'], + ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'], + ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Favorite Girl', 'music.album.artist', 'Justin Bieber'], + ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'], + ['Britney Spears', 'people.person.profession', 'Singer'], + ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'], + ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'], + ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'], + ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Rock music', + 'base.webvideo.internet_video_genre.series', + 'Biscuithands, The Animated Musical'], + ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Chef Tone', 'music.artist.genre', 'Hip hop music'], + ['Rudolph Isley', 'people.person.gender', 'Male'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'Barry Weiss'], + ['Beauty and a Beat (Bisbetic Instrumental)', + 'common.topic.notable_types', + 'Musical Recording'], + ['MTV Europe Music Award for Best Male', + 'award.award_category.winners', + 'm.0z1scxk'], + ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'], + ['Will Smith', 'people.person.profession', 'Actor'], + ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'], + ['Will i Am', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Boyfriend', 'music.composition.recordings', 'Boyfriend'], + ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'], + ['Fabian', 'people.person.gender', 'Male'], + ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'], + ['Somebody to Love (remix)', + 'music.album.primary_release', + 'Somebody to Love (remix)'], + ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'], + ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'], + ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'], + ['#thatPOWER', 'music.single.versions', '#thatPOWER'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'], + ['Spouse', 'type.property.expected_type', 'Person'], + ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'], + ['Baby', 'music.recording.artist', 'Ludacris'], + ['Rudolph Valentino', + 'people.person.nationality', + 'United States of America'], + ['Hit-Boy', 'music.artist.genre', 'Hip hop music'], + ['Judy Garland', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Kelly Clarkson', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['#thatPower', 'music.composition.recordings', '#Thatpower'], + ["Justin Bieber's Believe", + 'base.schemastaging.context_name.pronunciation', + 'm.011h9_22'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'], + ['Miley Cyrus', 'people.person.profession', 'Musician'], + ['Justin Timberlake', 'people.person.gender', 'Male'], + ['#Thatpower', 'music.recording.tracks', '#thatPOWER'], + ['m.0vp8rhw', + 'music.recording_contribution.album', + 'Beauty and a Beat (Remixes)'], + ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'], + ['Katy Perry: Part of Me', + 'common.topic.notable_types', + 'Award-Winning Work'], + ['m.0jsmvv5', + 'film.film_regional_release_date.film', + 'Justin Bieber: Never Say Never'], + ["Justin Bieber's Believe", 'common.topic.notable_for', 'g.1yj4hbf4k'], + ['My Worlds: The Collection', 'music.album.release_type', 'Album'], + ['All Around The World (featuring Ludacris)', + 'music.recording.artist', + 'Justin Bieber'], + ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'], + ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'], + ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'], + ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'], + ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'], + ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'], + ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'], + ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'], + ['School Boy Records', 'music.record_label.artist', 'Madison Beer'], + ["Justin Bieber's Believe", 'film.film.other_crew', 'm.0101ftl5'], + ['Musical Album', 'freebase.type_hints.included_types', 'Topic'], + ['As Long As You Love Me', + 'music.single.versions', + 'As Long As You Love Me (Audien dubstep mix)'], + ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'], + ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'], + ['Fergie', 'people.person.profession', 'Actor'], + ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Stuart Ford', 'people.person.profession', 'Film Producer'], + ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'], + ['Zac Efron', 'people.person.gender', 'Male'], + ['P!nk', 'music.artist.genre', 'Rock music'], + ['R. Kelly', 'people.person.profession', 'Film Producer'], + ['Gender', 'type.property.schema', 'Person'], + ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'], + ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'], + ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'], + ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'], + ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'], + ['Under the Mistletoe', + 'freebase.valuenotation.is_reviewed', + 'Initial release date'], + ['Live My Life', 'music.recording.tracks', 'Live My Life'], + ['The Island Def Jam Music Group', + 'music.record_label.artist', + 'Slick Rick'], + ['Amerie', 'music.artist.genre', 'Rock music'], + ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['m.0pbzq13', + 'film.performance.special_performance_type', + 'Cameo appearance'], + ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'], + ['Height', 'type.property.unit', 'Meter'], + ['Iggy Azalea', 'people.person.profession', 'Model'], + ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'], + ['Ray J', + 'freebase.valuenotation.has_no_value', + 'Spouse (or domestic partner)'], + ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'], + ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Boyfriend (acoustic version)', + 'music.recording.canonical_version', + 'Boyfriend'], + ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'], + ['Believe Tour', + 'music.concert_tour.album_or_release_supporting', + 'Believe'], + ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Believe Acoustic', 'music.album.release_type', 'Album'], + ['Diplo', 'freebase.valuenotation.has_value', 'Height'], + ['Hikaru Utada', 'music.artist.genre', 'Synthpop'], + ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'], + ['Frank Ocean', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['As Long As You Love Me (Audiobot instrumental)', + 'music.recording.song', + 'As Long as You Love Me'], + ['Elvis Presley', 'music.artist.genre', 'Pop music'], + ['Lady Gaga', 'music.artist.genre', 'Pop music'], + ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'], + ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'], + ['Usher', 'people.person.nationality', 'United States of America'], + ['Live My Life', 'music.composition.recordings', 'Live My Life'], + ['Kelis', 'music.artist.genre', 'Contemporary R&B'], + ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv5f'], + ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Next to You', 'music.recording.tracks', 'Next to You'], + ['m.0gbm3b7', + 'film.personal_film_appearance.type_of_appearance', + 'Him/Herself'], + ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Sheryl Crow', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'], + ['All That Matters', + 'music.composition.composer', + 'Jason \\"Poo Bear\\" Boyd'], + ['Nasri', 'music.artist.genre', 'Reggae'], + ['#thatPOWER', 'music.recording.song', '#thatPower'], + ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'], + ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'], + ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'], + ['Bad 25', 'film.film.genre', 'Documentary film'], + ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'], + ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'], + ['As Long as You Love Me', + 'music.composition.recordings', + 'As Long As You Love Me'], + ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'], + ['Geri Halliwell', 'people.person.profession', 'Musician'], + ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'], + ['Bryan-Michael Cox', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], + ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'], + ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'], + ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'], + ['Coldplay', 'music.artist.genre', 'Rock music'], + ['Kevin Risto', 'people.person.profession', 'Record producer'], + ['Whitney Houston', 'people.person.profession', 'Model'], + ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'], + ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], + ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'], + ['As Long as You Love Me', + 'music.recording.canonical_version', + 'As Long As You Love Me'], + ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Disney Parks Christmas Day Parade', + 'common.topic.notable_types', + 'Award-Winning Work'], + ['Ray J', 'people.person.profession', 'Artist'], + ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'], + ['American Music Award for Favorite Pop/Rock Male Artist', + 'award.award_category.winners', + 'm.0ndc0sf'], + ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'], + ['The Notorious B.I.G.', + 'freebase.valuenotation.is_reviewed', + 'Place of birth'], + ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'], + ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Christina Aguilera', 'music.artist.genre', 'Electronic music'], + ['PowerHitz', 'broadcast.content.artist', 'Outkast'], + ['U Smile', 'music.music_video.artist', 'Justin Bieber'], + ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'], + ['Sean Kingston', 'music.artist.genre', 'Hip hop music'], + ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['Haley James Scott', + 'fictional_universe.fictional_character.occupation', + 'Record producer'], + ['Kylie Minogue', 'music.artist.genre', 'Rock music'], + ['Chris Jasper', 'people.person.nationality', 'United States of America'], + ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['My Worlds: The Collection', + 'music.album.album_content_type', + 'Compilation album'], + ['Lolly', 'music.album.releases', 'Lolly'], + ['Toby Gad', 'common.topic.notable_types', 'Record Producer'], + ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'], + ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'], + ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'], + ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'], + ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'], + ['Ja Rule', 'people.person.profession', 'Singer'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'], + ['Die in Your Arms', + 'award.award_nominated_work.award_nominations', + 'm.0z85qxq'], + ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'], + ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'], + ['Kuk Harrell', + 'film.person_or_entity_appearing_in_film.films', + 'm.0101ft5f'], + ['Somebody to Love (J Stax remix)', + 'music.recording.artist', + 'Justin Bieber'], + ["Justin Bieber's Believe", + 'film.film.executive_produced_by', + 'Allison Kaye Scarinzi'], + ['Adam Messinger', 'people.person.nationality', 'Canada'], + ['Nasri', 'music.artist.genre', 'Pop music'], + ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'], + ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'], + ['C1', 'common.topic.notable_types', 'Musical Artist'], + ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'], + ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'], + ['Country', 'freebase.type_profile.strict_included_types', 'Topic'], + ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'], + ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['m.05sp405', + 'organization.organization_relationship.child', + 'Island Records'], + ['Savan Kotecha', 'people.person.profession', 'Record producer'], + ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'], + ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'], + ['John Mamann', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['Teen Choice Award for Choice Summer Music Star: Male', + 'award.award_category.winners', + 'm.0yrjynf'], + ['Juicy J', 'people.person.profession', 'Actor'], + ['m.0yqflrk', + 'measurement_unit.dated_money_value.source', + 'celebritynetworth.com'], + ['Miley Cyrus', + 'freebase.valuenotation.is_reviewed', + 'Country of nationality'], + ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'], + ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'], + ['m.04q65lb', + 'organization.organization_relationship.child', + 'The Island Def Jam Music Group'], + ['Big Sean', 'people.person.nationality', 'United States of America'], + ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'], + ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'], + ['1.FM Top 40', 'broadcast.content.artist', '\\"Weird Al\\" Yankovic'], + ['Geri Halliwell', 'people.person.profession', 'Actor'], + ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'], + ['My World', 'music.album.artist', 'Justin Bieber'], + ['Don Henley', 'people.person.gender', 'Male'], + ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'], + ['Musician', 'people.profession.specializations', 'Singer'], + ['Die in Your Arms', + 'music.recording.canonical_version', + 'Die in Your Arms'], + ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'], + ['m.0njvs9s', + 'award.award_honor.award', + 'CMT Music Award: Collaborative Video of the Year'], + ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'], + ['Justin Bieber', + 'music.artist.album', + 'Turn to You (Mother’s Day Dedication)'], + ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'], + ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'], + ['City/Town/Village', + 'freebase.type_profile.strict_included_types', + 'Topic'], + ['Recovery', 'common.topic.notable_types', 'Musical Recording'], + ['Dancer', 'common.topic.notable_types', 'Profession'], + ['Live My Life', 'common.topic.notable_types', 'Musical Recording'], + ['Terence Dudley', 'people.person.gender', 'Male'], + ['Baby', 'music.composition.recordings', 'Polka Face'], + ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['BeirutNights.com Radio', + 'broadcast.content.artist', + 'Mr. Sosa & The Yayo'], + ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'], + ['Rihanna', 'music.artist.genre', 'Dance music'], + ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'], + ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'], + ['Gender', 'type.property.expected_type', 'Gender'], + ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['m.0101fvbf', + 'film.film_regional_release_date.film', + "Justin Bieber's Believe"], + ['m.0yrhrwc', + 'award.award_honor.ceremony', + '2011 MTV Video Music Aid Japan'], + ['MTV Europe Music Award for Best North American Act', + 'award.award_category.winners', + 'm.0yrhmll'], + ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'], + ['m.0101ft1d', + 'film.personal_film_appearance.type_of_appearance', + 'Him/Herself'], + ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'], + ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'], + ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'], + ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'], + ['Ciara', 'people.person.gender', 'Female'], + ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'], + ['Britney Spears', 'music.artist.genre', 'Synthpop'], + ['Thought of You', 'music.recording.artist', 'Justin Bieber'], + ['m.0jzrrqs', + 'location.mailing_address.country', + 'United States of America'], + ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'], + ['Live My Life', 'music.composition.recordings', 'Live My Life'], + ['Toby Gad', 'people.person.nationality', 'United States of America'], + ['Big R Radio - Top 40 Hits', + 'broadcast.content.artist', + 'Natasha Bedingfield'], + ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'], + ...], + 'choices': []} + + + +Although this dataset can be trained on as-is, a couple problems emerge +from doing so: 1. A retrieval algorithm needs to be implemented and +executed during inference time, that might not appropriately correspond +to the algorithm that was used to generate the dataset subgraphs. 2. The +dataset as is not stored computationally efficiently, as there will +exist many duplicate nodes and edges that are shared between the +questions. + +As a result, it makes sense in this scenario to be able to encode all +the entries into a large knowledge graph, so that duplicate nodes and +edges can be avoided, and so that alternative retrieval algorithms can +be tried. We can do this with the LargeGraphIndexer class: + +.. code:: ipython3 + + from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups + from torch_geometric.nn.nlp import SentenceTransformer + import time + import torch + import tqdm + from itertools import chain + import networkx as nx + +.. code:: ipython3 + + raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']] + print(raw_dataset_graphs[0][:10]) + + +.. parsed-literal:: + + [('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell')] + + +To show the benefits of this indexer in action, we will use the +following model to encode this sample of graphs using LargeGraphIndexer, +along with naively. + +.. code:: ipython3 + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + print(device) + + +.. parsed-literal:: + + cuda + + +First, we compare the clock times of encoding using both methods. + +.. code:: ipython3 + + # Indexing question-by-question + dataset_graphs_embedded = [] + start = time.time() + for graph in tqdm.tqdm(raw_dataset_graphs): + nodes_map = dict() + edges_map = dict() + edge_idx_base = [] + + for src, edge, dst in graph: + # Collect nodes + if src not in nodes_map: + nodes_map[src] = len(nodes_map) + if dst not in nodes_map: + nodes_map[dst] = len(nodes_map) + + # Collect edge types + if edge not in edges_map: + edges_map[edge] = len(edges_map) + + # Record edge + edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst])) + + # Encode nodes and edges + sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x])) + sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x])) + + x = model.encode(sorted_nodes, batch_size=256) + edge_attrs_map = model.encode(sorted_edges, batch_size=256) + + edge_attrs = [] + edge_idx = [] + for trip in edge_idx_base: + edge_attrs.append(edge_attrs_map[trip[1]]) + edge_idx.append([trip[0], trip[2]]) + + dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0))) + + + print(time.time()-start) + + +.. parsed-literal:: + + 100%|██████████| 100/100 [02:01<00:00, 1.22s/it] + +.. parsed-literal:: + + 121.68579435348511 + + +.. parsed-literal:: + + + + +.. code:: ipython3 + + # Using LargeGraphIndexer to make one large knowledge graph + from torch_geometric.data.large_graph_indexer import EDGE_RELATION + + start = time.time() + all_triplets_together = chain.from_iterable(raw_dataset_graphs) + # Index as one large graph + print('Indexing...') + indexer = LargeGraphIndexer.from_triplets(all_triplets_together) + + # first the nodes + unique_nodes = indexer.get_unique_node_features() + node_encs = model.encode(unique_nodes, batch_size=256) + indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs) + + # then the edges + unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION) + edge_attr = model.encode(unique_edges, batch_size=256) + indexer.add_edge_feature(new_feature_name="edge_attr", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION) + + ckpt_time = time.time() + whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') + whole_graph_done = time.time() + print(f"Time to create whole knowledge_graph: {whole_graph_done-start}") + + # Compute this to make sure we're comparing like to like on final time printout + whole_graph_diff = whole_graph_done-ckpt_time + + # retrieve subgraphs + print('Retrieving Subgraphs...') + dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)] + print(time.time()-start-whole_graph_diff) + + +.. parsed-literal:: + + Indexing... + Time to create whole knowledge_graph: 114.01080107688904 + Retrieving Subgraphs... + + +.. parsed-literal:: + + 100%|██████████| 100/100 [00:00<00:00, 212.87it/s] + 100%|██████████| 100/100 [00:01<00:00, 80.90it/s] + +.. parsed-literal:: + + 114.66037964820862 + + +.. parsed-literal:: + + + + +The large graph indexer allows us to compute the entire knowledge graph +from a series of samples, so that new retrieval methods can also be +tested on the entire graph. We will see this attempted in practice later +on. + +It’s worth noting that, although the times are relatively similar right +now, the speedup with largegraphindexer will be much higher as the size +of the knowledge graph grows. This is due to the speedup being a factor +of the number of unique nodes and edges in the graph. + +.. code:: ipython3 + + dataset_graphs_embedded_largegraphindexer + + + + +.. parsed-literal:: + + [Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]), + Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]), + Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]), + Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]), + Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]), + Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]), + Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]), + Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]), + Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]), + Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]), + Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]), + Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]), + Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]), + Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]), + Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]), + Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]), + Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]), + Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]), + Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]), + Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]), + Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]), + Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]), + Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]), + Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]), + Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]), + Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]), + Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]), + Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]), + Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]), + Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]), + Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]), + Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]), + Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]), + Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]), + Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]), + Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]), + Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]), + Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]), + Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]), + Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]), + Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]), + Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]), + Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]), + Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]), + Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]), + Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]), + Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]), + Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]), + Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]), + Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]), + Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]), + Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]), + Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]), + Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]), + Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]), + Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]), + Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]), + Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]), + Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]), + Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]), + Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]), + Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]), + Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]), + Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]), + Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]), + Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]), + Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]), + Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]), + Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]), + Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]), + Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]), + Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]), + Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]), + Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]), + Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]), + Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]), + Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]), + Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]), + Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]), + Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]), + Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]), + Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]), + Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]), + Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]), + Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]), + Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]), + Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]), + Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]), + Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]), + Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]), + Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]), + Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]), + Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]), + Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]), + Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]), + Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]), + Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]), + Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]), + Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]), + Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])] + + + +.. code:: ipython3 + + dataset_graphs_embedded + + + + +.. parsed-literal:: + + [Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]), + Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]), + Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]), + Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]), + Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]), + Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]), + Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]), + Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]), + Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]), + Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]), + Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]), + Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]), + Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]), + Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]), + Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]), + Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]), + Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]), + Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]), + Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]), + Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]), + Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]), + Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]), + Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]), + Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]), + Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]), + Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]), + Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]), + Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]), + Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]), + Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]), + Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]), + Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]), + Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]), + Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]), + Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]), + Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]), + Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]), + Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]), + Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]), + Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]), + Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]), + Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]), + Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]), + Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]), + Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]), + Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]), + Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]), + Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]), + Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]), + Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]), + Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]), + Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]), + Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]), + Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]), + Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]), + Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]), + Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]), + Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]), + Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]), + Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]), + Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]), + Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]), + Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]), + Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]), + Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]), + Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]), + Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]), + Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]), + Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]), + Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]), + Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]), + Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]), + Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]), + Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]), + Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]), + Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]), + Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]), + Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]), + Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]), + Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]), + Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]), + Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]), + Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]), + Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]), + Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]), + Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]), + Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]), + Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]), + Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]), + Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]), + Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]), + Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]), + Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]), + Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]), + Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]), + Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]), + Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]), + Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]), + Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]), + Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])] + + + +We expect the two results to be functionally identical, with the +differences being due to floating point jitter. + +.. code:: ipython3 + + def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8): + def _sorted_tensors_are_close(tensor1, tensor2): + return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh) + def _graphs_are_same(tensor1, tensor2): + return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T)) + return _sorted_tensors_are_close(ground_truth.x, new_method.x) \ + and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \ + and _graphs_are_same(ground_truth.edge_index, new_method.edge_index) + +.. code:: ipython3 + + all_results_match = True + for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions): + all_results_match &= results_are_close_enough(old_graph, new_graph) + all_results_match + + +.. parsed-literal:: + + 100%|██████████| 100/100 [00:25<00:00, 4.00it/s] + + + + +.. parsed-literal:: + + True + + + +When scaled up to the entire dataset, we see a 2x speedup with indexing +this way. + +WebQSPDataset is a question-by-question implementation. + +UpdatedQSPDataset is a LargeGraphIndexer implementation. + +These were computed on an RTX 4090 with 24GB of memory. Your milage may +vary. + +Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph +---------------------------------------------------------------------------------------- + +Motivation +~~~~~~~~~~ + +One potential application of knowledge graph structural encodings is +capturing the relationships between different entities that are multiple +hops apart. This can be challenging for an LLM to recognize from +prepended graph information. Here’s a motivating example (credit to +@Rishi Puri): + +.. code:: ipython3 + + from IPython.display import SVG + +.. code:: ipython3 + + SVG(filename='./media/multihop_example.svg') + + + + +.. image:: 0_1_Encoding_from_Scratch_files/0_1_Encoding_from_Scratch_6_0.svg + + + +In this example, the question can only be answered by reasoning about +the relationships between the entities in the knowledge graph. + +Building a Multi-Hop QA Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To start, we need to download the raw data of a knowledge graph. In this +case, we use WikiData5M (`Wang et +al `__). +Here we download the raw triplets and their entity codes. Information +about this dataset can be found +`here `__. + +The following download contains the ID to plaintext mapping for all the +entities and relations in the knowledge graph: + +.. code:: ipython3 + + !wget -O "https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz" + +.. code:: ipython3 + + !tar -xvf "wikidata5m_alias.tar.gz" + +.. code:: ipython3 + + with open('wikidata5m_entity.txt') as f: + print(f.readline()) + +.. code:: ipython3 + + with open('wikidata5m_relation.txt') as f: + print(f.readline()) + +And then this download contains the raw triplets: + +.. code:: ipython3 + + !wget -O "https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz" + +.. code:: ipython3 + + !gzip -d "wikidata5m_all_triplet.txt.gz" -f + +.. code:: ipython3 + + with open('wikidata5m_all_triplet.txt') as f: + print(f.readline()) + +To start, we are going to preprocess the knowledge graph to substitute +each of the entity/relation codes with their plaintext aliases. This +makes it easier to use a pre-trained textual encoding model to create +triplet embeddings, as such a model likely won’t understand how to +properly embed the entity codes. + +.. code:: ipython3 + + import pandas as pd + import tqdm + import json + +.. code:: ipython3 + + # Substitute entity codes with their aliases + # Picking the first alias for each entity (rather arbitrarily) + alias_map = {} + rel_alias_map = {} + for line in open('wikidata5m_entity.txt'): + parts = line.strip().split('\t') + entity_id = parts[0] + aliases = parts[1:] + alias_map[entity_id] = aliases[0] + for line in open('wikidata5m_relation.txt'): + parts = line.strip().split('\t') + relation_id = parts[0] + relation_name = parts[1] + rel_alias_map[relation_id] = relation_name + +.. code:: ipython3 + + full_graph = [] + missing_total = 0 + total = 0 + for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')): + src, rel, dst = line.strip().split('\t') + if src not in alias_map: + missing_total += 1 + if dst not in alias_map: + missing_total += 1 + if rel not in rel_alias_map: + missing_total += 1 + total += 3 + full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)]) + print(f"Missing aliases: {missing_total}/{total}") + +.. code:: ipython3 + + full_graph[:10] + +Now ``full_graph`` represents the knowledge graph triplets in +understandable plaintext. + +Next, we need a set of multi-hop questions that the Knowledge Graph will +provide us with context for. We utilize a subset of +`HotPotQA `__ (`Yang et. +al. `__) called +`2WikiMultiHopQA `__ (`Ho et. +al. `__), which +includes a subgraph of entities that serve as the ground truth +justification for answering each multi-hop question: + +.. code:: ipython3 + + !wget -O "https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip" + +.. code:: ipython3 + + !unzip -o "data_ids_april7.zip" + +.. code:: ipython3 + + with open('train.json') as f: + train_data = json.load(f) + train_df = pd.DataFrame(train_data) + train_df['split_type'] = 'train' + + with open('dev.json') as f: + dev_data = json.load(f) + dev_df = pd.DataFrame(dev_data) + dev_df['split_type'] = 'dev' + + with open('test.json') as f: + test_data = json.load(f) + test_df = pd.DataFrame(test_data) + test_df['split_type'] = 'test' + + df = pd.concat([train_df, dev_df, test_df]) + +.. code:: ipython3 + + df.head() + +.. code:: ipython3 + + df['split_type'].value_counts() + +.. code:: ipython3 + + df['type'].value_counts() + +Now we need to extract the subgraphs + +.. code:: ipython3 + + df['graph_size'] = df['evidences_id'].apply(lambda row: len(row)) + +.. code:: ipython3 + + df['graph_size'].value_counts() + +(Optional) We take only questions where the evidence graph is greater +than 0. (Note: this gets rid of the test set): + +.. code:: ipython3 + + # df = df[df['graph_size'] > 0] + +.. code:: ipython3 + + df['split_type'].value_counts() + +.. code:: ipython3 + + df.columns + +.. code:: ipython3 + + refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']] + +.. code:: ipython3 + + refined_df.head() + +Checkpoint: + +.. code:: ipython3 + + refined_df.to_csv('wikimultihopqa_refined.csv', index=False) + +Now we need to check that all the entities mentioned in the +question/answer set are also present in the Wikidata graph: + +.. code:: ipython3 + + relation_map = {} + with open('wikidata5m_relation.txt') as f: + for line in tqdm.tqdm(f): + parts = line.strip().split('\t') + for i in range(1, len(parts)): + if parts[i] not in relation_map: + relation_map[parts[i]] = [] + relation_map[parts[i]].append(parts[0]) + +.. code:: ipython3 + + # Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M + for row in refined_df.itertuples(): + for trip in row.evidences_id: + relation = trip[1] + if relation not in relation_map: + print(f'The following relation is not found: {relation}') + elif len(relation_map[relation]) > 1: + print(f'The following relation alias has a collision: {relation}: {relation_map[relation]}') + +.. code:: ipython3 + + entity_set = set() + with open('wikidata5m_entity.txt') as f: + for line in tqdm.tqdm(f): + entity_set.add(line.strip().split('\t')[0]) + +.. code:: ipython3 + + missing_entities = set() + missing_entity_idx = set() + for i, row in enumerate(refined_df.itertuples()): + for trip in row.evidences_id: + if len(trip) != 3: + print(trip) + entities = trip[0], trip[2] + for entity in entities: + if entity not in entity_set: + print(f'The following entity was not found in the KG: {entity}') + missing_entities.add(entity) + missing_entity_idx.add(i) + +Right now, we drop the missing entity entries. Additional preprocessing +can be done here to resolve the entity/relation collisions, but that is +out of the scope for this notebook. + +.. code:: ipython3 + + len(missing_entity_idx) + +.. code:: ipython3 + + refined_df.shape + +.. code:: ipython3 + + # missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped. + refined_df.reset_index(inplace=True, drop=True) + refined_df + +.. code:: ipython3 + + cleaned_df = refined_df.drop(missing_entity_idx) + cleaned_df['split_type'].value_counts() + +Now we save the resulting graph and questions/answers dataset: + +.. code:: ipython3 + + cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False) + +.. code:: ipython3 + + import torch + +.. code:: ipython3 + + torch.save(full_graph, 'wikimultihopqa_full_graph.pt') + +Question: How do we extract a contextual subgraph for a given query? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The chosen retrieval algorithm is a critical component in the pipeline +for affecting RAG performance. In the next section (1), we will +demonstrate a naive method of retrieval for a large knowledge graph, and +how to apply it to this dataset along with WebQSP. + +Preparing a Textualized Graph for LLM +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For now however, we need to prepare the graph data to be used as a +plaintext prefix to the LLM. In order to do this, we want to prompt the +LLM to use the unique nodes, and unique edge triplets of a given +subgraph. In order to do this, we prepare a unique indexed node df and +edge df for the knowledge graph now. This process occurs trivially with +the LargeGraphIndexer: + +.. code:: ipython3 + + from torch_geometric.data import LargeGraphIndexer + +.. code:: ipython3 + + indexer = LargeGraphIndexer.from_triplets(full_graph) + +.. code:: ipython3 + + # Node DF + textual_nodes = pd.DataFrame.from_dict( + {"node_attr": indexer.get_node_features()}) + textual_nodes["node_id"] = textual_nodes.index + textual_nodes = textual_nodes[["node_id", "node_attr"]] + +.. code:: ipython3 + + textual_nodes.head() + +Notice how LargeGraphIndexer ensures that there are no duplicate +indices: + +.. code:: ipython3 + + textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0] + +.. code:: ipython3 + + # Edge DF + textual_edges = pd.DataFrame(indexer.get_edge_features(), + columns=["src", "edge_attr", "dst"]) + textual_edges["src"] = [ + indexer._nodes[h] for h in textual_edges["src"] + ] + textual_edges["dst"] = [ + indexer._nodes[h] for h in textual_edges["dst"] + ] + +Note: The edge table refers to each node by its index in the node table. +We will see how this gets utilized later when indexing a subgraph. + +.. code:: ipython3 + + textual_edges.head() + +Now we can save the result + +.. code:: ipython3 + + textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False) + textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False) + +Now were done! This knowledge graph and dataset will get used later on +in Section 1. + +.. code:: ipython3 + + # TODO: Refactor everything below this point into its own notebook + +Generating Subgraphs +==================== + +.. code:: ipython3 + + from profiling_utils import create_remote_backend_from_triplets + from rag_feature_store import SentenceTransformerFeatureStore + from rag_graph_store import NeighborSamplingRAGGraphStore + from torch_geometric.loader import RAGQueryLoader + from torch_geometric.nn.nlp import SentenceTransformer + from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst + from torch_geometric.data import get_features_for_triplets_groups, Data + +.. code:: ipython3 + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) + +.. code:: ipython3 + + fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={"batch_size": 256}).load() + +.. code:: ipython3 + + import torch + import pandas as pd + +.. code:: ipython3 + + graphs = torch.load('subg_results.pt') + graph_df = pd.read_csv("wikimultihopqa_cleaned.csv") + +.. code:: ipython3 + + graph_df['is_train'].value_counts() + +.. code:: ipython3 + + graph_df.head() + +.. code:: ipython3 + + graph_df['is_train'].value_counts() + + +Retrieval Algorithms and Scaling Retrieval +========================================== + +Motivation +---------- + +When building a RAG Pipeline for inference, the retrieval component is +important for the following reasons: 1. A given algorithm for retrieving +subgraph context can have a marked effect on the hallucination rate of +the responses in the model 2. A given retrieval algorithm needs to be +able to scale to larger graphs of millions of nodes and edges in order +to be practical for production. + +In this notebook, we will explore how to construct a RAG retrieval +algorithm from a given subgraph, and conduct some experiments to +evaluate its runtime performance. + +We want to do so in-line with Pytorch Geometric’s in-house framework for +remote backends: + +.. code:: ipython3 + + from IPython.display import Image, SVG + Image(filename='../../../docs/source/_figures/remote_2.png') + + + + +.. image:: 1_Retrieval_files/1_Retrieval_5_0.png + + + +As seen here, the GraphStore is used to store the neighbor relations +between the nodes of the graph, whereas the FeatureStore is used to +store the node and edge features in the graph. + +Let’s start by loading in a knowledge graph dataset for the sake of our +experiment: + +.. code:: ipython3 + + from torch_geometric.data import LargeGraphIndexer + from torch_geometric.datasets import UpdatedWebQSPDataset + from itertools import chain + + +.. parsed-literal:: + + /home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html + from .autonotebook import tqdm as notebook_tqdm + + +.. code:: ipython3 + + # Limiting to 10 questions for the sake of compute, but can be increased if necessary + ds = UpdatedWebQSPDataset(root='demo', limit=10) + +Let’s set up our set of questions and graph triplets: + +.. code:: ipython3 + + questions = ds.raw_dataset['question'] + questions + + + + +.. parsed-literal:: + + ['what is the name of justin bieber brother', + 'what character did natalie portman play in star wars', + 'what country is the grand bahama island in', + 'what kind of money to take to bahamas', + 'what character did john noble play in lord of the rings', + 'who does joakim noah play for', + 'where are the nfl redskins from', + 'where did saki live', + 'who did draco malloy end up marrying', + 'which countries border the us'] + + + +.. code:: ipython3 + + ds.raw_dataset[:10]['graph'][0][:10] + + + + +.. parsed-literal:: + + [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'], + ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'], + ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'], + ['Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'], + ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'], + ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'], + ['Stephen Melton', 'people.person.nationality', 'United States of America'], + ['Record producer', + 'music.performance_role.regular_performances', + 'm.012m1vf1'], + ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'], + ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell']] + + + +.. code:: ipython3 + + all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset)) + +With these questions and triplets, we want to: 1. Consolidate all the +relations in these triplets into a Knowledge Graph 2. Create a +FeatureStore that encodes all the nodes and edges in the knowledge graph +3. Create a GraphStore that encodes all the edge indices in the +knowledge graph + +.. code:: ipython3 + + import torch + from torch_geometric.nn.nlp import SentenceTransformer + from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet + +.. code:: ipython3 + + import sys + sys.path.append('..') + +In order to create a remote backend, we need to define a FeatureStore +and GraphStore locally, as well as a method for initializing its state +from triplets: + +.. code:: ipython3 + + from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader + + # We define this GraphStore to sample the neighbors of a node locally. + # Ideally for a real remote backend, this interface would be replaced with an API to a Graph DB, such as Neo4j. + from rag_graph_store import NeighborSamplingRAGGraphStore + + # We define this FeatureStore to encode the nodes and edges locally, and perform appoximate KNN when indexing. + # Ideally for a real remote backend, this interface would be replaced with an API to a vector DB, such as Pinecone. + from rag_feature_store import SentenceTransformerFeatureStore + +.. code:: ipython3 + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SentenceTransformer(model_name="sentence-transformers/all-roberta-large-v1").to(device) + + backend_loader: RemoteGraphBackendLoader = create_remote_backend_from_triplets( + triplets=all_triplets, # All the triplets to insert into the backend + node_embedding_model=model, # Embedding model to process triplets with + node_method_to_call="encode", # This method will encode the nodes/edges with 'model.encode' in this case. + path="backend", # Save path + pre_transform=preprocess_triplet, # Preprocessing function to apply to triplets before invoking embedding model. + node_method_kwargs={"batch_size": 256}, # Keyword arguments to pass to the node_method_to_call. + graph_db=NeighborSamplingRAGGraphStore, # Graph Store to use + feature_db=SentenceTransformerFeatureStore # Feature Store to use + ) + # This loader saves a copy of the processed data locally to be transformed into a graphstore and featurestore when load() is called. + feature_store, graph_store = backend_loader.load() + +Now that we have initialized our remote backends, we can now retrieve +from them using a Loader to query the backends, as shown in this +diagram: + +.. code:: ipython3 + + Image(filename='../../../docs/source/_figures/remote_3.png') + + + + +.. image:: 1_Retrieval_files/1_Retrieval_21_0.png + + + +.. code:: ipython3 + + from torch_geometric.loader import RAGQueryLoader + +.. code:: ipython3 + + query_loader = RAGQueryLoader( + data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store + # Arguments to pass into the seed node/edge retrieval methods for the FeatureStore. + # In this case, it's k for the KNN on the nodes and edges. + seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, + # Arguments to pass into the GraphStore's Neighbor sampling method. + # In this case, the GraphStore implements a NeighborLoader, so it takes the same arguments. + sampler_kwargs={"num_neighbors": [40]*3}, + # Arguments to pass into the FeatureStore's feature loading method. + loader_kwargs={}, + # An optional local transform that can be applied on the returned subgraph. + local_filter=None, + ) + +To make better sense of this loader’s arguments, let’s take a closer +look at the retrieval process for a remote backend: + +.. code:: ipython3 + + SVG(filename="media/remote_backend.svg") + + + + +.. image:: 1_Retrieval_files/1_Retrieval_25_0.svg + + + +As we see here, there are 3 important steps to any remote backend +procedure for graphs: 1. Retrieve the seed nodes and edges to begin our +retrieval process from. 2. Traverse the graph neighborhood of the seed +nodes/edges to gather local context. 3. Fetch the features associated +with the subgraphs obtained from the traversal. + +We can see that our Query Loader construction allows us to specify +unique hyperparameters for each unique step in this retrieval. + +Now we can submit our queries to the remote backend to retrieve our +subgraphs: + +.. code:: ipython3 + + import tqdm + +.. code:: ipython3 + + sub_graphs = [] + for q in tqdm.tqdm(questions): + sub_graphs.append(query_loader.query(q)) + + +.. parsed-literal:: + + 0%| | 0/10 [00:00 Date: Wed, 28 Aug 2024 14:25:18 -0700 Subject: [PATCH 685/752] reorder files --- .../llm_rag}/benchmark_model_archs.py | 0 .../llm_rag}/benchmark_model_gnn_enc.py | 0 examples/llm_plus_gnn/__init__.py | 0 examples/llm_plus_gnn/benchmark/__init__.py | 0 .../rag_generate_multihop.py | 0 .../{benchmark => }/rag_generate.py | 0 .../llm_plus_gnn/rag_generate_multihop_.py | 52 ------------------- 7 files changed, 52 deletions(-) rename {examples/llm_plus_gnn/benchmark => benchmark/llm_rag}/benchmark_model_archs.py (100%) rename {examples/llm_plus_gnn/benchmark => benchmark/llm_rag}/benchmark_model_gnn_enc.py (100%) delete mode 100644 examples/llm_plus_gnn/__init__.py delete mode 100644 examples/llm_plus_gnn/benchmark/__init__.py rename examples/llm_plus_gnn/{benchmark => multihop}/rag_generate_multihop.py (100%) rename examples/llm_plus_gnn/{benchmark => }/rag_generate.py (100%) delete mode 100644 examples/llm_plus_gnn/rag_generate_multihop_.py diff --git a/examples/llm_plus_gnn/benchmark/benchmark_model_archs.py b/benchmark/llm_rag/benchmark_model_archs.py similarity index 100% rename from examples/llm_plus_gnn/benchmark/benchmark_model_archs.py rename to benchmark/llm_rag/benchmark_model_archs.py diff --git a/examples/llm_plus_gnn/benchmark/benchmark_model_gnn_enc.py b/benchmark/llm_rag/benchmark_model_gnn_enc.py similarity index 100% rename from examples/llm_plus_gnn/benchmark/benchmark_model_gnn_enc.py rename to benchmark/llm_rag/benchmark_model_gnn_enc.py diff --git a/examples/llm_plus_gnn/__init__.py b/examples/llm_plus_gnn/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/llm_plus_gnn/benchmark/__init__.py b/examples/llm_plus_gnn/benchmark/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/llm_plus_gnn/benchmark/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py similarity index 100% rename from examples/llm_plus_gnn/benchmark/rag_generate_multihop.py rename to examples/llm_plus_gnn/multihop/rag_generate_multihop.py diff --git a/examples/llm_plus_gnn/benchmark/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py similarity index 100% rename from examples/llm_plus_gnn/benchmark/rag_generate.py rename to examples/llm_plus_gnn/rag_generate.py diff --git a/examples/llm_plus_gnn/rag_generate_multihop_.py b/examples/llm_plus_gnn/rag_generate_multihop_.py deleted file mode 100644 index 7b0df0dfb28c..000000000000 --- a/examples/llm_plus_gnn/rag_generate_multihop_.py +++ /dev/null @@ -1,52 +0,0 @@ -# %% -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import RAGQueryLoader -from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd - - -# %% -triplets = torch.load('wikimultihopqa_full_graph.pt') - -# %% -df = pd.read_csv('wikimultihopqa_cleaned.csv') -questions = df['question_text'].to_list() - -# %% -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) - -# %% -print("Generating remote backend...") -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() - -# %% -print("Retrieving subgraphs...") -df_textual_nodes = pd.read_csv('wikimultihopqa_textual_nodes.csv') -df_textual_edges = pd.read_csv('wikimultihopqa_textual_edges.csv') - -def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: - q_emb = model.encode(query) - textual_nodes = df_textual_nodes.iloc[graph["node_idx"]].reset_index() - textual_edges = df_textual_edges.iloc[graph["edge_idx"]].reset_index() - out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) - out_graph["desc"] = desc - return out_graph - -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*3}, local_filter=apply_retrieval_via_pcst) - -# %% -subgs = [] -for subg in tqdm.tqdm(query_loader.batch_query(questions)): - print(subg) - subgs.append(subg) - -torch.save(subgs, 'subg_results.pt') From 770a7d1e167a5359051d2677c7e10f15bbf18798 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 14:51:29 -0700 Subject: [PATCH 686/752] begin merger of uwebqsp and webqsp --- benchmark/llm_rag/benchmark_model_gnn_enc.py | 4 +- docs/source/advanced/rag.rst | 17 +- ...0_0_Encoding_a_Large_Knowledge_Graph.ipynb | 1798 +- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 19690 +--------------- examples/llm_plus_gnn/g_retriever.py | 10 +- .../nvtx_examples/nvtx_rag_backend_example.py | 4 +- .../nvtx_examples/nvtx_uwebqsp_example.py | 27 - test/datasets/test_web_qsp_dataset.py | 36 +- torch_geometric/datasets/__init__.py | 2 - .../datasets/updated_web_qsp_dataset.py | 357 - torch_geometric/datasets/web_qsp_dataset.py | 237 +- 11 files changed, 1949 insertions(+), 20233 deletions(-) delete mode 100644 examples/llm_plus_gnn/nvtx_examples/nvtx_uwebqsp_example.py delete mode 100644 torch_geometric/datasets/updated_web_qsp_dataset.py diff --git a/benchmark/llm_rag/benchmark_model_gnn_enc.py b/benchmark/llm_rag/benchmark_model_gnn_enc.py index 60c753aeba2e..d777d9f8dede 100644 --- a/benchmark/llm_rag/benchmark_model_gnn_enc.py +++ b/benchmark/llm_rag/benchmark_model_gnn_enc.py @@ -1,5 +1,5 @@ # %% -from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever from g_retriever import benchmark_models, get_loss, inference_step @@ -13,7 +13,7 @@ eval_batch_size = 16 # %% -ds = UpdatedWebQSPDataset('benchmark_archs') +ds = WebQSPDataset('benchmark_archs') # %% model_names = [] diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 9400f6de9514..2dffedb3707e 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -72,10 +72,10 @@ WebQSP: .. code:: ipython3 - from torch_geometric.datasets import UpdatedWebQSPDataset + from torch_geometric.datasets import WebQSPDataset num_questions = 100 - ds = UpdatedWebQSPDataset('small_sample', limit=num_questions) + ds = WebQSPDataset('small_sample', limit=num_questions) .. parsed-literal:: @@ -1920,14 +1920,7 @@ differences being due to floating point jitter. When scaled up to the entire dataset, we see a 2x speedup with indexing -this way. - -WebQSPDataset is a question-by-question implementation. - -UpdatedQSPDataset is a LargeGraphIndexer implementation. - -These were computed on an RTX 4090 with 24GB of memory. Your milage may -vary. +this way on the WebQSP Dataset. Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph ---------------------------------------------------------------------------------------- @@ -2394,7 +2387,7 @@ experiment: .. code:: ipython3 from torch_geometric.data import LargeGraphIndexer - from torch_geometric.datasets import UpdatedWebQSPDataset + from torch_geometric.datasets import WebQSPDataset from itertools import chain @@ -2407,7 +2400,7 @@ experiment: .. code:: ipython3 # Limiting to 10 questions for the sake of compute, but can be increased if necessary - ds = UpdatedWebQSPDataset(root='demo', limit=10) + ds = WebQSPDataset(root='demo', limit=10) Let’s set up our set of questions and graph triplets: diff --git a/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb b/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb index 0b1993b71dec..e1de6f2309c7 100644 --- a/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb +++ b/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb @@ -30,14 +30,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ - "from torch_geometric.datasets import WebQSPDataset, UpdatedWebQSPDataset\n", + "from torch_geometric.datasets import UpdatedWebQSPDataset\n", "\n", - "# Computationally expensive, so running a small sample for now to show off the schema\n", - "# ds = WebQSPDataset('dataset')\n", "num_questions = 100\n", "ds = UpdatedWebQSPDataset('small_sample', limit=num_questions)" ] @@ -54,18 +61,1405 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", + " num_rows: 100\n", + "})" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ds.raw_dataset" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'WebQTrn-0',\n", + " 'question': 'what is the name of justin bieber brother',\n", + " 'answer': ['Jaxon Bieber'],\n", + " 'q_entity': ['Justin Bieber'],\n", + " 'a_entity': ['Jaxon Bieber'],\n", + " 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", + " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", + " ['Rudolph Valentino',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", + " ['Record producer',\n", + " 'music.performance_role.regular_performances',\n", + " 'm.012m1vf1'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", + " ['2011 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0yrkr34'],\n", + " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Stratford', 'location.location.containedby', 'Canada'],\n", + " ['Singer',\n", + " 'base.lightweight.profession.specialization_of',\n", + " 'Musicians and Singers'],\n", + " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", + " ['As Long As You Love Me (Audien dubstep mix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kevin Risto', 'people.person.gender', 'Male'],\n", + " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", + " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", + " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", + " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", + " ['American Music Award for Favorite Pop/Rock Album',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc259'],\n", + " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", + " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", + " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Billboard Music Award for Top Streaming Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0njhx1b'],\n", + " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", + " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", + " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", + " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.episode',\n", + " 'Results Show: Week 7'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", + " ['One Less Lonely Girl',\n", + " 'music.album.primary_release',\n", + " 'One Less Lonely Girl'],\n", + " ['Twista', 'people.person.gender', 'Male'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", + " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", + " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", + " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", + " ['Madonna', 'music.artist.genre', 'Pop music'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", + " ['m.0gbm3cg',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", + " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", + " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", + " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", + " ['MTV Video Music Award Japan for Best New Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhrwc'],\n", + " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", + " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", + " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['m.0gctwjk',\n", + " 'tv.tv_guest_role.episodes_appeared_in',\n", + " 'Series 2, Episode 3'],\n", + " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", + " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", + " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", + " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", + " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Beauty and a Beat',\n", + " 'music.single.versions',\n", + " 'Beauty and a Beat (Wideboys Club Mix)'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Bryan Adams',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", + " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", + " ['Dany Brillant', 'people.person.gender', 'Male'],\n", + " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", + " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", + " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'music.composition.form', 'Song'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0yrkr34'],\n", + " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", + " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", + " ['Record producer',\n", + " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", + " 'Haley James Scott'],\n", + " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", + " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['Kanye West', 'people.person.profession', 'Singer'],\n", + " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", + " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", + " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", + " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", + " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", + " ['RedOne', 'music.artist.genre', 'Rock music'],\n", + " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", + " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", + " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['As Long As You Love Me (Ferry Corsten radio)',\n", + " 'music.recording.featured_artists',\n", + " 'Big Sean'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", + " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", + " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", + " ['Athan Grace', 'people.person.profession', 'Actor'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", + " ['All Around the World (acoustic version)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", + " ['Brandy Norwood',\n", + " 'influence.influence_node.influenced_by',\n", + " 'Whitney Houston'],\n", + " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['MTV Video Music Award for Artist to Watch',\n", + " 'award.award_category.winners',\n", + " 'm.0n1ykxp'],\n", + " ['Caitlin Beadles',\n", + " 'celebrities.celebrity.sexual_relationships',\n", + " 'm.0d33gyj'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audiobot instrumental)'],\n", + " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", + " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'The Mighty Mighty Bosstones'],\n", + " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", + " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", + " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Documentary film'],\n", + " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", + " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Amerie', 'music.artist.genre', 'Pop music'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", + " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep edit)'],\n", + " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Recovery', 'music.recording.song', 'Recovery'],\n", + " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", + " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", + " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", + " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", + " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", + " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", + " ['#thatPower', 'music.recording.song', '#thatPower'],\n", + " ['Children', 'rdf-schema#range', 'Person'],\n", + " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", + " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['2013 Teen Choice Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0wjgqck'],\n", + " ['The Island Def Jam Music Group',\n", + " 'organization.organization.parent',\n", + " 'm.04q65lb'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Rusted Root'],\n", + " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", + " ['m.0z87d3n',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", + " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", + " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", + " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", + " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", + " ['Tampa', 'location.location.containedby', 'United States of America'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.compositions',\n", + " 'Love Never Felt So Good'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", + " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", + " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", + " ['Estelle', 'people.person.profession', 'Record producer'],\n", + " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", + " ['Chris Jasper', 'people.person.gender', 'Male'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", + " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", + " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", + " ['Snoop Dogg',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['David Nicksay', 'people.person.gender', 'Male'],\n", + " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", + " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Juno Awards of 2014',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0102z0vx'],\n", + " ['As Long As You Love Me (Audiobot remix)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", + " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", + " ['m.0_cyzs_',\n", + " 'celebrities.legal_entanglement.offense',\n", + " 'Driving under the influence'],\n", + " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", + " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", + " ['Mann', 'people.person.gender', 'Male'],\n", + " ['JoJo', 'people.person.gender', 'Female'],\n", + " ['Right Here (featuring Drake)',\n", + " 'music.recording.canonical_version',\n", + " 'Right Here'],\n", + " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", + " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0yrjynf',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", + " ['Pras', 'people.person.profession', 'Record producer'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", + " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", + " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", + " ['Marvin Isley',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", + " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['School Gyrls', 'film.film.language', 'English Language'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", + " ['Katy Perry', 'people.person.profession', 'Actor'],\n", + " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", + " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.featured_artists',\n", + " 'Redfoo'],\n", + " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", + " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", + " ['As Long as You Love Me (album version)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['Justin Bieber: Just Getting Started',\n", + " 'book.written_work.author',\n", + " 'Justin Bieber'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Marc Maris vs. Ramone'],\n", + " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", + " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", + " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", + " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", + " ['Love Never Felt So Good',\n", + " 'music.album.releases',\n", + " 'Love Never Felt So Good'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", + " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", + " ['Dance music',\n", + " 'broadcast.genre.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", + " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", + " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", + " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", + " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", + " ['Scooter Braun', 'people.person.gender', 'Male'],\n", + " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", + " ['Sir Nolan', 'people.person.gender', 'Male'],\n", + " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", + " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", + " ['Adam Messinger',\n", + " 'music.composer.compositions',\n", + " \"Turn to You (Mother's Day Dedication)\"],\n", + " ['m.0yrktlv',\n", + " 'award.award_honor.award',\n", + " 'Teen Choice Award for Choice Male Hottie'],\n", + " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", + " ['Iggy Azalea',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", + " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", + " ['m.0gxnnzy',\n", + " 'celebrities.romantic_relationship.relationship_type',\n", + " 'Dated'],\n", + " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", + " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", + " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Britney Spears',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", + " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", + " ['Fergie', 'music.artist.genre', 'Rock music'],\n", + " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", + " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", + " ['Mistletoe', 'music.album.release_type', 'Single'],\n", + " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", + " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", + " ['Live My Life (Party Rock remix)',\n", + " 'music.recording.tracks',\n", + " 'Live My Life (Party Rock remix)'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['m.0njw4z2',\n", + " 'award.award_honor.award',\n", + " 'MTV Europe Music Award for Best Male'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", + " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", + " ['m.0gbm3c3',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Miley Cyrus'],\n", + " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", + " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", + " ['United States of America',\n", + " 'base.biblioness.bibs_topic.is_really',\n", + " 'United States of America'],\n", + " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", + " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", + " ['JoJo', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Next to You', 'music.album.release_type', 'Single'],\n", + " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", + " ['Willa Ford', 'people.person.languages', 'English Language'],\n", + " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", + " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", + " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", + " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Hot Wired Radio',\n", + " 'broadcast.content.broadcast',\n", + " 'Hot Wired Radio - 128kbps Stream'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", + " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['m.0gbm3d9',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", + " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", + " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", + " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", + " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", + " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", + " ['Justin Timberlake',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Khalil', 'people.person.gender', 'Male'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", + " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", + " ['Selena Gomez',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", + " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", + " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['C1', 'music.artist.genre', 'Hip hop music'],\n", + " ['My Worlds Acoustic',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Album content type'],\n", + " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", + " ['Live My Life', 'music.composition.language', 'English Language'],\n", + " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", + " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", + " ['Baby', 'music.album.releases', 'Baby'],\n", + " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", + " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", + " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", + " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", + " ['Big R Radio - The Hawk',\n", + " 'broadcast.content.artist',\n", + " 'The Black Eyed Peas'],\n", + " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", + " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", + " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", + " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.production_companies',\n", + " 'AEG Live'],\n", + " ['Redfoo', 'people.person.gender', 'Male'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", + " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", + " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", + " ['m.01053qzf',\n", + " 'film.personal_film_appearance.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", + " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", + " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", + " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", + " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", + " ['Ray J', 'people.person.profession', 'Musician'],\n", + " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", + " ['m.0101fv5f',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", + " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", + " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", + " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", + " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", + " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0v_729v',\n", + " 'tv.tv_guest_personal_appearance.appearance_type',\n", + " 'Guest host'],\n", + " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", + " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", + " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", + " ['Christina Milian', 'people.person.profession', 'Actor'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", + " ['Dance music', 'broadcast.genre.content', '181-party'],\n", + " ['Queen Elizabeth II Diamond Jubilee Medal',\n", + " 'award.award_category.winners',\n", + " 'm.0njwqrb'],\n", + " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", + " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", + " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['CMT Music Award: Collaborative Video of the Year',\n", + " 'award.award_category.winners',\n", + " 'm.0njvs9s'],\n", + " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", + " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", + " ['Chingy', 'people.person.profession', 'Actor'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", + " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", + " ['Justin Bieber',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", + " ['Chingy', 'people.person.gender', 'Male'],\n", + " ['Reed Smoot', 'people.person.gender', 'Male'],\n", + " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", + " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.releases',\n", + " 'As Long As You Love Me (remixes)'],\n", + " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjvlh'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", + " ['Singer', 'common.topic.article', 'm.09l6h'],\n", + " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", + " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'award.award_winning_work.awards_won',\n", + " 'm.0pc670l'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", + " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", + " ['Beyoncé Knowles',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['m.0njhyh_',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Streaming Song (Video)'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", + " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_production_design_by',\n", + " 'Devorah Herbert'],\n", + " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", + " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", + " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", + " ['Nick Jonas',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", + " ['Lupe Fiasco',\n", + " 'broadcast.artist.content',\n", + " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", + " ['Martin Kierszenbaum',\n", + " 'people.person.place_of_birth',\n", + " 'United States of America'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long as You Love Me'],\n", + " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", + " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", + " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", + " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", + " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", + " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['m.0y5tl39',\n", + " 'film.personal_film_appearance.film',\n", + " 'Les Coulisses des Golden Globes'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", + " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z0tgz6'],\n", + " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", + " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", + " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", + " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", + " ['Everlast', 'music.artist.label', 'Island Records'],\n", + " ['5th Annual Shorty Awards',\n", + " 'award.award_ceremony.awards_presented',\n", + " 'm.0ywvh8k'],\n", + " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Baby', 'common.topic.notable_types', 'Composition'],\n", + " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['m.0tkqqgg',\n", + " 'award.award_nomination.award',\n", + " 'Juno Award for Pop Album of the Year'],\n", + " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", + " ['Person', 'type.type.properties', 'Parents'],\n", + " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Nasri', 'people.person.profession', 'Singer'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", + " ['As Long as You Love Me',\n", + " 'music.album.compositions',\n", + " 'As Long as You Love Me'],\n", + " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", + " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", + " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", + " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", + " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", + " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", + " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", + " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", + " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", + " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", + " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", + " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.film',\n", + " 'Zendaya: Behind the Scenes'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", + " ['That Should Be Me', 'music.composition.form', 'Song'],\n", + " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", + " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Justin Bieber'],\n", + " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", + " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", + " ['Beauty and a Beat (acoustic version)',\n", + " 'music.recording.song',\n", + " 'Beauty And A Beat'],\n", + " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", + " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", + " ['Usher', 'music.composer.compositions', 'First Dance'],\n", + " ['m.0n1ykxp',\n", + " 'award.award_honor.award',\n", + " 'MTV Video Music Award for Artist to Watch'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'media_common.netflix_title.netflix_genres',\n", + " 'Rockumentary'],\n", + " ['Amerie', 'people.person.gender', 'Female'],\n", + " ['Real Change: Artists for Education',\n", + " 'film.film.personal_appearances',\n", + " 'm.0y5th3r'],\n", + " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", + " ['Beautiful and the Beat',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", + " ['PYD', 'common.topic.notable_types', 'Composition'],\n", + " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", + " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", + " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", + " ['iJustine', 'people.person.gender', 'Female'],\n", + " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", + " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", + " ['m.0ng_vkd',\n", + " 'film.personal_film_appearance.film',\n", + " 'A Michael Bublé Christmas'],\n", + " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", + " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", + " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", + " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Vanessa Hudgens',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", + " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", + " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", + " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", + " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", + " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", + " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Madonna', 'people.person.profession', 'Record producer'],\n", + " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", + " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", + " ['As Long as You Love Me (acoustic version)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", + " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", + " ['Rob Thomas', 'people.person.gender', 'Male'],\n", + " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", + " ['m.0hvlt03',\n", + " 'film.film_film_distributor_relationship.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", + " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", + " ['justinbieber',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_srv2b'],\n", + " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", + " ['Donna Summer',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['m.0101fszs',\n", + " 'film.personal_film_appearance.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['Alanis Morissette',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Official website'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Jenna Andrews'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", + " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Next to You', 'music.recording.song', 'Next to You'],\n", + " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Ray J', 'people.person.nationality', 'United States of America'],\n", + " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", + " ['m.0w3gbtv',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", + " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['Nelly Furtado',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", + " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", + " ['As Long As You Love Me (Ferry Corsten remix)',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['m.0gbm3fj',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", + " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", + " ['As Long As You Love Me (Ferry Corsten club dub)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", + " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", + " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", + " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", + " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", + " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", + " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", + " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", + " ['Guest host',\n", + " 'tv.non_character_role.tv_guest_personal_appearances',\n", + " 'm.0v_98y7'],\n", + " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", + " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", + " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", + " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", + " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", + " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", + " ['L.A. Reid', 'people.person.gender', 'Male'],\n", + " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", + " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['London', 'location.location.containedby', 'Ontario'],\n", + " ['Justin Bieber: Never Say Never',\n", + " 'film.film.film_set_decoration_by',\n", + " 'Lia Roldan'],\n", + " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", + " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", + " ['Children', 'type.property.schema', 'Person'],\n", + " ['Change Me', 'music.album.releases', 'Change Me'],\n", + " ['RedOne', 'music.artist.label', 'Island Records'],\n", + " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", + " ['All Around the World',\n", + " 'music.recording.canonical_version',\n", + " 'All Around the World'],\n", + " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", + " ['Teen Choice Award for Choice Twitter Personality',\n", + " 'award.award_category.winners',\n", + " 'm.0wjhc6c'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", + " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", + " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", + " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", + " 'broadcast.content.artist',\n", + " 'Lupe Fiasco'],\n", + " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", + " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", + " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", + " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", + " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", + " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Beauty and a Beat (Wideboys Radio Mix)',\n", + " 'music.recording.canonical_version',\n", + " 'Beauty and a Beat'],\n", + " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", + " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", + " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", + " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", + " ['m.0v90skf',\n", + " 'award.award_honor.award',\n", + " 'Billboard Music Award for Top Male Artist'],\n", + " ['Ludacris', 'people.person.profession', 'Actor'],\n", + " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", + " ['Cameo appearance',\n", + " 'tv.special_tv_performance_type.episode_performances',\n", + " 'm.0v1lwt2'],\n", + " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", + " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", + " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", + " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'One Chance'],\n", + " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", + " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", + " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", + " ['Roller Coaster',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0_x4zg3'],\n", + " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", + " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", + " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", + " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", + " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", + " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", + " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", + " ['m.0sgk_cw',\n", + " 'award.award_honor.award',\n", + " \"Kids' Choice Award for Favorite Song\"],\n", + " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", + " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", + " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", + " ['MTV Video Music Brazil Award for Best International Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhhqv'],\n", + " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", + " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", + " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", + " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", + " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", + " ['Britney Spears', 'people.person.profession', 'Singer'],\n", + " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", + " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", + " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Rock music',\n", + " 'base.webvideo.internet_video_genre.series',\n", + " 'Biscuithands, The Animated Musical'],\n", + " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", + " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Barry Weiss'],\n", + " ['Beauty and a Beat (Bisbetic Instrumental)',\n", + " 'common.topic.notable_types',\n", + " 'Musical Recording'],\n", + " ['MTV Europe Music Award for Best Male',\n", + " 'award.award_category.winners',\n", + " 'm.0z1scxk'],\n", + " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", + " ['Will Smith', 'people.person.profession', 'Actor'],\n", + " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", + " ['Will i Am',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", + " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", + " ['Fabian', 'people.person.gender', 'Male'],\n", + " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", + " ['Somebody to Love (remix)',\n", + " 'music.album.primary_release',\n", + " 'Somebody to Love (remix)'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", + " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", + " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", + " ['Spouse', 'type.property.expected_type', 'Person'],\n", + " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", + " ['Baby', 'music.recording.artist', 'Ludacris'],\n", + " ['Rudolph Valentino',\n", + " 'people.person.nationality',\n", + " 'United States of America'],\n", + " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", + " ['Judy Garland',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Kelly Clarkson',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'base.schemastaging.context_name.pronunciation',\n", + " 'm.011h9_22'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", + " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", + " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", + " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", + " ['m.0vp8rhw',\n", + " 'music.recording_contribution.album',\n", + " 'Beauty and a Beat (Remixes)'],\n", + " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", + " ['Katy Perry: Part of Me',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['m.0jsmvv5',\n", + " 'film.film_regional_release_date.film',\n", + " 'Justin Bieber: Never Say Never'],\n", + " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", + " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", + " ['All Around The World (featuring Ludacris)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", + " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", + " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", + " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", + " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", + " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", + " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", + " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", + " ['As Long As You Love Me',\n", + " 'music.single.versions',\n", + " 'As Long As You Love Me (Audien dubstep mix)'],\n", + " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", + " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Fergie', 'people.person.profession', 'Actor'],\n", + " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", + " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", + " ['Zac Efron', 'people.person.gender', 'Male'],\n", + " ['P!nk', 'music.artist.genre', 'Rock music'],\n", + " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", + " ['Gender', 'type.property.schema', 'Person'],\n", + " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", + " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", + " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", + " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", + " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", + " ['Under the Mistletoe',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Initial release date'],\n", + " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", + " ['The Island Def Jam Music Group',\n", + " 'music.record_label.artist',\n", + " 'Slick Rick'],\n", + " ['Amerie', 'music.artist.genre', 'Rock music'],\n", + " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.0pbzq13',\n", + " 'film.performance.special_performance_type',\n", + " 'Cameo appearance'],\n", + " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", + " ['Height', 'type.property.unit', 'Meter'],\n", + " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", + " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", + " ['Ray J',\n", + " 'freebase.valuenotation.has_no_value',\n", + " 'Spouse (or domestic partner)'],\n", + " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", + " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Boyfriend (acoustic version)',\n", + " 'music.recording.canonical_version',\n", + " 'Boyfriend'],\n", + " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", + " ['Believe Tour',\n", + " 'music.concert_tour.album_or_release_supporting',\n", + " 'Believe'],\n", + " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", + " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", + " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", + " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", + " ['Frank Ocean',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['As Long As You Love Me (Audiobot instrumental)',\n", + " 'music.recording.song',\n", + " 'As Long as You Love Me'],\n", + " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", + " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", + " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", + " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", + " ['Usher', 'people.person.nationality', 'United States of America'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", + " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", + " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", + " ['m.0gbm3b7',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['Sheryl Crow',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", + " ['All That Matters',\n", + " 'music.composition.composer',\n", + " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", + " ['Nasri', 'music.artist.genre', 'Reggae'],\n", + " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", + " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", + " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", + " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", + " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", + " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", + " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['As Long as You Love Me',\n", + " 'music.composition.recordings',\n", + " 'As Long As You Love Me'],\n", + " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", + " ['Bryan-Michael Cox',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", + " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", + " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", + " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", + " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", + " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", + " ['Whitney Houston', 'people.person.profession', 'Model'],\n", + " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", + " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", + " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", + " ['As Long as You Love Me',\n", + " 'music.recording.canonical_version',\n", + " 'As Long As You Love Me'],\n", + " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Disney Parks Christmas Day Parade',\n", + " 'common.topic.notable_types',\n", + " 'Award-Winning Work'],\n", + " ['Ray J', 'people.person.profession', 'Artist'],\n", + " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", + " ['American Music Award for Favorite Pop/Rock Male Artist',\n", + " 'award.award_category.winners',\n", + " 'm.0ndc0sf'],\n", + " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", + " ['The Notorious B.I.G.',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Place of birth'],\n", + " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", + " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", + " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", + " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", + " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", + " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", + " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['Haley James Scott',\n", + " 'fictional_universe.fictional_character.occupation',\n", + " 'Record producer'],\n", + " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", + " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", + " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['My Worlds: The Collection',\n", + " 'music.album.album_content_type',\n", + " 'Compilation album'],\n", + " ['Lolly', 'music.album.releases', 'Lolly'],\n", + " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", + " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", + " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", + " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", + " ['Ja Rule', 'people.person.profession', 'Singer'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", + " ['Die in Your Arms',\n", + " 'award.award_nominated_work.award_nominations',\n", + " 'm.0z85qxq'],\n", + " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", + " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", + " ['Kuk Harrell',\n", + " 'film.person_or_entity_appearing_in_film.films',\n", + " 'm.0101ft5f'],\n", + " ['Somebody to Love (J Stax remix)',\n", + " 'music.recording.artist',\n", + " 'Justin Bieber'],\n", + " [\"Justin Bieber's Believe\",\n", + " 'film.film.executive_produced_by',\n", + " 'Allison Kaye Scarinzi'],\n", + " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", + " ['Nasri', 'music.artist.genre', 'Pop music'],\n", + " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", + " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", + " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", + " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", + " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", + " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", + " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", + " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", + " ['m.05sp405',\n", + " 'organization.organization_relationship.child',\n", + " 'Island Records'],\n", + " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", + " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", + " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", + " ['John Mamann',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['Teen Choice Award for Choice Summer Music Star: Male',\n", + " 'award.award_category.winners',\n", + " 'm.0yrjynf'],\n", + " ['Juicy J', 'people.person.profession', 'Actor'],\n", + " ['m.0yqflrk',\n", + " 'measurement_unit.dated_money_value.source',\n", + " 'celebritynetworth.com'],\n", + " ['Miley Cyrus',\n", + " 'freebase.valuenotation.is_reviewed',\n", + " 'Country of nationality'],\n", + " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", + " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['m.04q65lb',\n", + " 'organization.organization_relationship.child',\n", + " 'The Island Def Jam Music Group'],\n", + " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", + " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", + " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", + " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", + " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", + " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", + " ['My World', 'music.album.artist', 'Justin Bieber'],\n", + " ['Don Henley', 'people.person.gender', 'Male'],\n", + " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", + " ['Musician', 'people.profession.specializations', 'Singer'],\n", + " ['Die in Your Arms',\n", + " 'music.recording.canonical_version',\n", + " 'Die in Your Arms'],\n", + " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", + " ['m.0njvs9s',\n", + " 'award.award_honor.award',\n", + " 'CMT Music Award: Collaborative Video of the Year'],\n", + " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", + " ['Justin Bieber',\n", + " 'music.artist.album',\n", + " 'Turn to You (Mother’s Day Dedication)'],\n", + " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", + " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", + " ['City/Town/Village',\n", + " 'freebase.type_profile.strict_included_types',\n", + " 'Topic'],\n", + " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", + " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", + " ['Terence Dudley', 'people.person.gender', 'Male'],\n", + " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", + " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['BeirutNights.com Radio',\n", + " 'broadcast.content.artist',\n", + " 'Mr. Sosa & The Yayo'],\n", + " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", + " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", + " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", + " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", + " ['Gender', 'type.property.expected_type', 'Gender'],\n", + " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", + " ['m.0101fvbf',\n", + " 'film.film_regional_release_date.film',\n", + " \"Justin Bieber's Believe\"],\n", + " ['m.0yrhrwc',\n", + " 'award.award_honor.ceremony',\n", + " '2011 MTV Video Music Aid Japan'],\n", + " ['MTV Europe Music Award for Best North American Act',\n", + " 'award.award_category.winners',\n", + " 'm.0yrhmll'],\n", + " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", + " ['m.0101ft1d',\n", + " 'film.personal_film_appearance.type_of_appearance',\n", + " 'Him/Herself'],\n", + " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", + " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", + " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", + " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", + " ['Ciara', 'people.person.gender', 'Female'],\n", + " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", + " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", + " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", + " ['m.0jzrrqs',\n", + " 'location.mailing_address.country',\n", + " 'United States of America'],\n", + " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", + " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", + " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", + " ['Big R Radio - Top 40 Hits',\n", + " 'broadcast.content.artist',\n", + " 'Natasha Bedingfield'],\n", + " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", + " ...],\n", + " 'choices': []}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ds.raw_dataset[0]" ] @@ -88,7 +1482,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -103,12 +1497,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell')]\n" + ] + } + ], "source": [ "raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']]\n", - "print(raw_dataset_graphs[0])" + "print(raw_dataset_graphs[0][:10])" ] }, { @@ -120,9 +1522,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuda\n" + ] + } + ], "source": [ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)\n", @@ -138,9 +1548,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [02:01<00:00, 1.22s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "121.68579435348511\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "# Indexing question-by-question\n", "dataset_graphs_embedded = []\n", @@ -185,9 +1617,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Indexing...\n", + "Time to create whole knowledge_graph: 114.01080107688904\n", + "Retrieving Subgraphs...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 212.87it/s]\n", + "100%|██████████| 100/100 [00:01<00:00, 80.90it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "114.66037964820862\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "# Using LargeGraphIndexer to make one large knowledge graph\n", "from torch_geometric.data.large_graph_indexer import EDGE_RELATION\n", @@ -238,18 +1702,238 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]),\n", + " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]),\n", + " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]),\n", + " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]),\n", + " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]),\n", + " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]),\n", + " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]),\n", + " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]),\n", + " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]),\n", + " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]),\n", + " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]),\n", + " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]),\n", + " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]),\n", + " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]),\n", + " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]),\n", + " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]),\n", + " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", + " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]),\n", + " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]),\n", + " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]),\n", + " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]),\n", + " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]),\n", + " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]),\n", + " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]),\n", + " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]),\n", + " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]),\n", + " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]),\n", + " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]),\n", + " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]),\n", + " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]),\n", + " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]),\n", + " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]),\n", + " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]),\n", + " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]),\n", + " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]),\n", + " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", + " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]),\n", + " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]),\n", + " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]),\n", + " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]),\n", + " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]),\n", + " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]),\n", + " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]),\n", + " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]),\n", + " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]),\n", + " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]),\n", + " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]),\n", + " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]),\n", + " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]),\n", + " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]),\n", + " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]),\n", + " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]),\n", + " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]),\n", + " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]),\n", + " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]),\n", + " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]),\n", + " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]),\n", + " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]),\n", + " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]),\n", + " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]),\n", + " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]),\n", + " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]),\n", + " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]),\n", + " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]),\n", + " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]),\n", + " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]),\n", + " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]),\n", + " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]),\n", + " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]),\n", + " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]),\n", + " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", + " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", + " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]),\n", + " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]),\n", + " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]),\n", + " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]),\n", + " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]),\n", + " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]),\n", + " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]),\n", + " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]),\n", + " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]),\n", + " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]),\n", + " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]),\n", + " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]),\n", + " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", + " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", + " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]),\n", + " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dataset_graphs_embedded_largegraphindexer" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]),\n", + " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]),\n", + " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]),\n", + " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]),\n", + " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]),\n", + " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]),\n", + " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]),\n", + " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]),\n", + " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]),\n", + " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]),\n", + " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]),\n", + " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]),\n", + " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]),\n", + " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]),\n", + " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]),\n", + " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]),\n", + " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", + " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]),\n", + " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]),\n", + " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]),\n", + " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]),\n", + " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]),\n", + " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]),\n", + " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]),\n", + " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]),\n", + " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]),\n", + " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]),\n", + " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]),\n", + " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]),\n", + " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]),\n", + " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]),\n", + " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]),\n", + " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]),\n", + " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]),\n", + " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]),\n", + " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", + " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]),\n", + " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]),\n", + " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]),\n", + " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]),\n", + " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]),\n", + " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]),\n", + " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]),\n", + " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]),\n", + " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]),\n", + " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]),\n", + " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]),\n", + " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]),\n", + " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]),\n", + " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]),\n", + " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]),\n", + " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]),\n", + " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]),\n", + " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]),\n", + " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]),\n", + " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]),\n", + " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]),\n", + " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]),\n", + " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]),\n", + " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]),\n", + " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]),\n", + " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]),\n", + " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]),\n", + " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]),\n", + " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]),\n", + " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]),\n", + " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]),\n", + " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]),\n", + " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", + " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]),\n", + " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", + " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]),\n", + " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]),\n", + " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]),\n", + " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]),\n", + " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]),\n", + " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]),\n", + " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]),\n", + " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]),\n", + " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]),\n", + " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]),\n", + " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]),\n", + " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]),\n", + " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]),\n", + " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]),\n", + " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]),\n", + " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", + " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]),\n", + " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", + " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]),\n", + " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dataset_graphs_embedded" ] @@ -263,7 +1947,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -279,9 +1963,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:25<00:00, 4.00it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "all_results_match = True\n", "for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions):\n", @@ -302,50 +2004,6 @@ "These were computed on an RTX 4090 with 24GB of memory. Your milage may vary." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "demo_number_of_graphs = 4700 # out of 4700" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Note: WebQSP doesn't let you limit the number of questions, so we compute the whole dataset and estimate the time by assuming each question takes the same amount of time\n", - "start = time.time()\n", - "ds_basic = WebQSPDataset('basic_dataset')\n", - "web_qsp_time = (time.time() - start)*demo_number_of_graphs/4700\n", - "print(time.time()-start)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "start = time.time()\n", - "ds_updated = UpdatedWebQSPDataset('updated_dataset', limit=demo_number_of_graphs)\n", - "updated_time = time.time() - start\n", - "print(time.time()-start)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "plt.bar(['WebQSP', 'UpdatedQSP'], [web_qsp_time,, updated_time])" - ] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb index c3f8e37873a6..17778a50b125 100644 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -49,7 +49,7 @@ "" ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -75,9 +75,18 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from torch_geometric.data import LargeGraphIndexer\n", "from torch_geometric.datasets import UpdatedWebQSPDataset\n", @@ -86,50 +95,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Saving the dataset (1/1 shards): 100%|██████████| 10/10 [00:00<00:00, 923.25 examples/s] \n", - "Processing...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Encoding graph...\n", - "Saving graph...\n", - "Encoding questions...\n", - "Retrieving subgraphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "10it [00:00, 31.85it/s]00:00 thresh) - - def _graphs_are_same(tensor1, tensor2): - return nx.weisfeiler_lehman_graph_hash(nx.Graph( - tensor1.T)) == nx.weisfeiler_lehman_graph_hash( - nx.Graph(tensor2.T)) - return _sorted_tensors_are_close(ground_truth.x, new_method.x) \ - and _sorted_tensors_are_close(ground_truth.edge_attr, - new_method.edge_attr) \ - and _graphs_are_same(ground_truth.edge_index, - new_method.edge_index) - - dataset_original = WebQSPDataset() - dataset_updated = UpdatedWebQSPDataset('updated') - - for ground_truth, updated_graph in zip(dataset_original, dataset_updated): - assert results_are_close_enough(ground_truth, updated_graph) +def test_web_qsp_dataset_limit(): + dataset = WebQSPDataset(limit=100) + assert len(dataset) == 100 + assert str(dataset) == "WebQSPDataset(100)" \ No newline at end of file diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index 1c45d30abf21..cc8ef498debb 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -77,7 +77,6 @@ from .myket import MyketDataset from .brca_tgca import BrcaTcga from .web_qsp_dataset import WebQSPDataset -from .updated_web_qsp_dataset import UpdatedWebQSPDataset from .dbp15k import DBP15K from .aminer import AMiner @@ -112,7 +111,6 @@ import torch_geometric.datasets.utils homo_datasets = [ - 'UpdatedWebQSPDataset', 'WebQSPDataset', 'KarateClub', 'TUDataset', diff --git a/torch_geometric/datasets/updated_web_qsp_dataset.py b/torch_geometric/datasets/updated_web_qsp_dataset.py deleted file mode 100644 index fac429d845d6..000000000000 --- a/torch_geometric/datasets/updated_web_qsp_dataset.py +++ /dev/null @@ -1,357 +0,0 @@ -import os -from itertools import chain -from typing import Iterator, List, Tuple, no_type_check - -import numpy as np -import torch -from tqdm import tqdm - -from torch_geometric.data import ( - Data, - InMemoryDataset, - LargeGraphIndexer, - TripletLike, - get_features_for_triplets_groups, -) -from torch_geometric.data.large_graph_indexer import EDGE_RELATION -from torch_geometric.nn.nlp import SentenceTransformer - -try: - from pandas import DataFrame - WITH_PANDAS = True -except ImportError: - DataFrame = None - WITH_PANDAS = False - -try: - from pcst_fast import pcst_fast - - WITH_PCST = True -except ImportError: - WITH_PCST = False - -try: - import datasets - - WITH_DATASETS = True -except ImportError: - WITH_DATASETS = False - - -@no_type_check -def retrieval_via_pcst( - graph: Data, - q_emb: torch.Tensor, - textual_nodes: DataFrame, - textual_edges: DataFrame, - topk: int = 3, - topk_e: int = 3, - cost_e: float = 0.5, - save_idx: bool = False, - override: bool = False, -) -> Tuple[Data, str]: - c = 0.01 - if len(textual_nodes) == 0 or len(textual_edges) == 0 or override: - desc = (textual_nodes.to_csv(index=False) + - "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"])) - graph = Data( - x=graph.x, - edge_index=graph.edge_index, - edge_attr=graph.edge_attr, - num_nodes=graph.num_nodes, - ) - return graph, desc - - root = -1 # unrooted - num_clusters = 1 - pruning = "gw" - verbosity_level = 0 - if topk > 0: - n_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.x) - topk = min(topk, graph.num_nodes) - _, topk_n_indices = torch.topk(n_prizes, topk, largest=True) - - n_prizes = torch.zeros_like(n_prizes) - n_prizes[topk_n_indices] = torch.arange(topk, 0, -1).float() - else: - n_prizes = torch.zeros(graph.num_nodes) - - if topk_e > 0: - e_prizes = torch.nn.CosineSimilarity(dim=-1)(q_emb, graph.edge_attr) - topk_e = min(topk_e, e_prizes.unique().size(0)) - - topk_e_values, _ = torch.topk(e_prizes.unique(), topk_e, largest=True) - e_prizes[e_prizes < topk_e_values[-1]] = 0.0 - last_topk_e_value = topk_e - for k in range(topk_e): - indices = e_prizes == topk_e_values[k] - value = min((topk_e - k) / sum(indices), last_topk_e_value - c) - e_prizes[indices] = value - last_topk_e_value = value - # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) - else: - e_prizes = torch.zeros(graph.num_edges) - - costs = [] - edges = [] - virtual_n_prizes = [] - virtual_edges = [] - virtual_costs = [] - mapping_n = {} - mapping_e = {} - for i, (src, dst) in enumerate(graph.edge_index.T.numpy()): - prize_e = e_prizes[i] - if prize_e <= cost_e: - mapping_e[len(edges)] = i - edges.append((src, dst)) - costs.append(cost_e - prize_e) - else: - virtual_node_id = graph.num_nodes + len(virtual_n_prizes) - mapping_n[virtual_node_id] = i - virtual_edges.append((src, virtual_node_id)) - virtual_edges.append((virtual_node_id, dst)) - virtual_costs.append(0) - virtual_costs.append(0) - virtual_n_prizes.append(prize_e - cost_e) - - prizes = np.concatenate([n_prizes, np.array(virtual_n_prizes)]) - num_edges = len(edges) - if len(virtual_costs) > 0: - costs = np.array(costs + virtual_costs) - edges = np.array(edges + virtual_edges) - - vertices, edges = pcst_fast(edges, prizes, costs, root, num_clusters, - pruning, verbosity_level) - - selected_nodes = vertices[vertices < graph.num_nodes] - selected_edges = [mapping_e[e] for e in edges if e < num_edges] - virtual_vertices = vertices[vertices >= graph.num_nodes] - if len(virtual_vertices) > 0: - virtual_vertices = vertices[vertices >= graph.num_nodes] - virtual_edges = [mapping_n[i] for i in virtual_vertices] - selected_edges = np.array(selected_edges + virtual_edges) - - edge_index = graph.edge_index[:, selected_edges] - selected_nodes = np.unique( - np.concatenate( - [selected_nodes, edge_index[0].numpy(), edge_index[1].numpy()])) - - n = textual_nodes.iloc[selected_nodes] - e = textual_edges.iloc[selected_edges] - desc = (n.to_csv(index=False) + "\n" + - e.to_csv(index=False, columns=["src", "edge_attr", "dst"])) - - mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} - - x = graph.x[selected_nodes] - edge_attr = graph.edge_attr[selected_edges] - src = [mapping[i] for i in edge_index[0].tolist()] - dst = [mapping[i] for i in edge_index[1].tolist()] - edge_index = torch.LongTensor([src, dst]) - data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(selected_nodes)) - - # HACK Added so that the subset of nodes and edges selected can be tracked - if save_idx: - data['node_idx'] = np.array(graph.node_idx)[selected_nodes] - data['edge_idx'] = np.array(graph.edge_idx)[selected_edges] - - return data, desc - - -def preprocess_triplet(triplet: TripletLike) -> TripletLike: - h, r, t = triplet - return str(h).lower(), str(r), str(t).lower() - - -class UpdatedWebQSPDataset(InMemoryDataset): - def __init__( - self, - root: str = "", - force_reload: bool = False, - whole_graph_retrieval: bool = False, - limit: int = -1, - override: bool = False, - ) -> None: - """Construct a WebQSPDataset. - - Args: - root (str, optional): Path to save dataset to. Defaults to "". - force_reload (bool, optional): Whether to rerun process step. - Defaults to False. - whole_graph_retrieval (bool, optional): Whether to retrieve the - entire knowledge graph at once. Defaults to False. - limit (int, optional): Construct only the first n samples. - Defaults to -1 to construct all samples. - override (bool, optional): Whether to skip WebQSP step. Defaults - to False. - """ - self.limit = limit - self.whole_graph_retrieval = whole_graph_retrieval - self.override = override - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") - super().__init__(root, None, None, force_reload=force_reload) - self._load_raw_data() - self.load(self.processed_paths[0]) - - def _check_dependencies(self) -> None: - missing_str_list = [] - if not WITH_PCST: - missing_str_list.append('pcst_fast') - if not WITH_DATASETS: - missing_str_list.append('datasets') - if not WITH_PANDAS: - missing_str_list.append('pandas') - if len(missing_str_list) > 0: - missing_str = ' '.join(missing_str_list) - error_out = f"`pip install {missing_str}` to use this dataset." - raise ImportError(error_out) - - @property - def raw_file_names(self) -> List[str]: - return ["raw_data", "split_idxs"] - - @property - def processed_file_names(self) -> List[str]: - return [ - "list_of_graphs.pt", - "pre_filter.pt", - "pre_transform.pt", - "large_graph_indexer", - ] - - def _save_raw_data(self) -> None: - self.raw_dataset.save_to_disk(self.raw_paths[0]) - torch.save(self.split_idxs, self.raw_paths[1]) - - def _load_raw_data(self) -> None: - if not hasattr(self, "raw_dataset"): - self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) - if not hasattr(self, "split_idxs"): - self.split_idxs = torch.load(self.raw_paths[1]) - - def download(self) -> None: - dataset = datasets.load_dataset("rmanluo/RoG-webqsp") - self.raw_dataset = datasets.concatenate_datasets( - [dataset["train"], dataset["validation"], dataset["test"]]) - self.split_idxs = { - "train": - torch.arange(len(dataset["train"])), - "val": - torch.arange(len(dataset["validation"])) + len(dataset["train"]), - "test": - torch.arange(len(dataset["test"])) + len(dataset["train"]) + - len(dataset["validation"]), - } - - if self.limit >= 0: - self.raw_dataset = self.raw_dataset.select(range(self.limit)) - self.split_idxs = { - "train": - torch.arange(self.limit // 2), - "val": - torch.arange(self.limit // 4) + self.limit // 2, - "test": - torch.arange(self.limit // 4) + self.limit // 2 + - self.limit // 4, - } - self._save_raw_data() - - def _get_trips(self) -> Iterator[TripletLike]: - return chain.from_iterable( - iter(ds["graph"]) for ds in self.raw_dataset) - - def _build_graph(self) -> None: - trips = self._get_trips() - self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( - trips, pre_transform=preprocess_triplet) - - # Nodes: - nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256) # type: ignore - self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) - - # Edges: - edges = self.indexer.get_unique_edge_features( - feature_name=EDGE_RELATION) - edge_attr = self.model.encode(edges, batch_size=256) # type: ignore - self.indexer.add_edge_feature( - new_feature_name="edge_attr", - new_feature_vals=edge_attr, - map_from_feature=EDGE_RELATION, - ) - - print("Saving graph...") - self.indexer.save(self.processed_paths[-1]) - - def _retrieve_subgraphs(self) -> None: - print("Encoding questions...") - self.questions = [str(ds["question"]) for ds in self.raw_dataset] - q_embs = self.model.encode(self.questions, batch_size=256) - list_of_graphs = [] - print("Retrieving subgraphs...") - textual_nodes = self.textual_nodes - textual_edges = self.textual_edges - if self.whole_graph_retrieval: - graph = self.indexer.to_data(node_feature_name="x", - edge_feature_name="edge_attr") - else: - graph_gen = get_features_for_triplets_groups( - self.indexer, (ds['graph'] for ds in self.raw_dataset), - pre_transform=preprocess_triplet) - - for index in tqdm(range(len(self.raw_dataset))): - data_i = self.raw_dataset[index] - if not self.whole_graph_retrieval: - graph = next(graph_gen) - textual_nodes = self.textual_nodes.iloc[ - graph["node_idx"]].reset_index() - textual_edges = self.textual_edges.iloc[ - graph["edge_idx"]].reset_index() - pcst_subgraph, desc = retrieval_via_pcst( - graph, - q_embs[index], - textual_nodes, - textual_edges, - topk=3, - topk_e=5, - cost_e=0.5, - override=self.override, - ) - question = f"Question: {data_i['question']}\nAnswer: " - label = ("|").join(data_i["answer"]).lower() - - pcst_subgraph["question"] = question - pcst_subgraph["label"] = label - pcst_subgraph["desc"] = desc - list_of_graphs.append(pcst_subgraph.to("cpu")) - print("Saving subgraphs...") - self.save(list_of_graphs, self.processed_paths[0]) - - def process(self) -> None: - self._load_raw_data() - self.model = SentenceTransformer( - 'sentence-transformers/all-roberta-large-v1').to(self.device) - self.model.eval() - if not os.path.exists(self.processed_paths[-1]): - print("Encoding graph...") - self._build_graph() - else: - print("Loading graph...") - self.indexer = LargeGraphIndexer.from_disk( - self.processed_paths[-1]) - self.textual_nodes = DataFrame.from_dict( - {"node_attr": self.indexer.get_node_features()}) - self.textual_nodes["node_id"] = self.textual_nodes.index - self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] - self.textual_edges = DataFrame(self.indexer.get_edge_features(), - columns=["src", "edge_attr", "dst"]) - self.textual_edges["src"] = [ - self.indexer._nodes[h] for h in self.textual_edges["src"] - ] - self.textual_edges["dst"] = [ - self.indexer._nodes[h] for h in self.textual_edges["dst"] - ] - self._retrieve_subgraphs() diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 82cc574d73f7..03204960e3a7 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,12 +1,19 @@ -# Alot of code in this file is based on the original G-Retriever paper -# url: https://arxiv.org/abs/2402.07630 -from typing import Dict, List, Tuple, no_type_check +import os +from itertools import chain +from typing import Iterator, List, Tuple, no_type_check import numpy as np import torch from tqdm import tqdm -from torch_geometric.data import Data, InMemoryDataset +from torch_geometric.data import ( + Data, + InMemoryDataset, + LargeGraphIndexer, + TripletLike, + get_features_for_triplets_groups, +) +from torch_geometric.data.large_graph_indexer import EDGE_RELATION from torch_geometric.nn.nlp import SentenceTransformer try: @@ -18,12 +25,14 @@ try: from pcst_fast import pcst_fast + WITH_PCST = True except ImportError: WITH_PCST = False try: import datasets + WITH_DATASETS = True except ImportError: WITH_DATASETS = False @@ -38,13 +47,20 @@ def retrieval_via_pcst( topk: int = 3, topk_e: int = 3, cost_e: float = 0.5, + save_idx: bool = False, + override: bool = False, ) -> Tuple[Data, str]: c = 0.01 - if len(textual_nodes) == 0 or len(textual_edges) == 0: - desc = textual_nodes.to_csv(index=False) + "\n" + textual_edges.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) - graph = Data(x=graph.x, edge_index=graph.edge_index, - edge_attr=graph.edge_attr, num_nodes=graph.num_nodes) + if len(textual_nodes) == 0 or len(textual_edges) == 0 or override: + desc = (textual_nodes.to_csv(index=False) + + "\n" + textual_edges.to_csv( + index=False, columns=["src", "edge_attr", "dst"])) + graph = Data( + x=graph.x, + edge_index=graph.edge_index, + edge_attr=graph.edge_attr, + num_nodes=graph.num_nodes, + ) return graph, desc root = -1 # unrooted @@ -72,9 +88,8 @@ def retrieval_via_pcst( indices = e_prizes == topk_e_values[k] value = min((topk_e - k) / sum(indices), last_topk_e_value - c) e_prizes[indices] = value - last_topk_e_value = value * (1 - c) - # reduce the cost of the edges such that at least one edge is selected - cost_e = min(cost_e, e_prizes.max().item() * (1 - c / 2)) + last_topk_e_value = value + # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) else: e_prizes = torch.zeros(graph.num_edges) @@ -124,8 +139,8 @@ def retrieval_via_pcst( n = textual_nodes.iloc[selected_nodes] e = textual_edges.iloc[selected_edges] - desc = n.to_csv(index=False) + "\n" + e.to_csv( - index=False, columns=["src", "edge_attr", "dst"]) + desc = (n.to_csv(index=False) + "\n" + + e.to_csv(index=False, columns=["src", "edge_attr", "dst"])) mapping = {n: i for i, n in enumerate(selected_nodes.tolist())} @@ -137,10 +152,20 @@ def retrieval_via_pcst( data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, num_nodes=len(selected_nodes)) + # HACK Added so that the subset of nodes and edges selected can be tracked + if save_idx: + data['node_idx'] = np.array(graph.node_idx)[selected_nodes] + data['edge_idx'] = np.array(graph.edge_idx)[selected_edges] + return data, desc -class WebQSPDataset(InMemoryDataset): +def preprocess_triplet(triplet: TripletLike) -> TripletLike: + h, r, t = triplet + return str(h).lower(), str(r), str(t).lower() + + +class UpdatedWebQSPDataset(InMemoryDataset): r"""The WebQuestionsSP dataset was released as part of “The Value of Semantic Parse Labeling for Knowledge Base Question Answering” @@ -150,21 +175,31 @@ class WebQSPDataset(InMemoryDataset): Processing based on "G-Retriever: Retrieval-Augmented Generation for Textual Graph Understanding and Question Answering". Requires datasets and transformers from HuggingFace. - - Args: - root (str): Root directory where the dataset should be saved. - force_reload (bool, optional): Whether to re-process the dataset. - (default: :obj:`False`) """ def __init__( self, root: str = "", force_reload: bool = False, + limit: int = -1, + include_pcst: bool = True, ) -> None: - self._check_dependencies() + """Construct a WebQSPDataset. + + Args: + root (str, optional): Path to save dataset to. Defaults to "". + force_reload (bool, optional): Whether to rerun process step. + Defaults to False. + limit (int, optional): Construct only the first n samples. + Defaults to -1 to construct all samples. + include_pcst (bool, optional): Whether to include PCST step + (See GRetriever paper). Defaults to True. + """ + self.limit = limit + self.include_pcst = include_pcst self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") super().__init__(root, None, None, force_reload=force_reload) + self._load_raw_data() self.load(self.processed_paths[0]) def _check_dependencies(self) -> None: @@ -182,11 +217,26 @@ def _check_dependencies(self) -> None: @property def raw_file_names(self) -> List[str]: - return [] + return ["raw_data", "split_idxs"] @property def processed_file_names(self) -> List[str]: - return ["list_of_graphs.pt", "pre_filter.pt", "pre_transform.pt"] + return [ + "list_of_graphs.pt", + "pre_filter.pt", + "pre_transform.pt", + "large_graph_indexer", + ] + + def _save_raw_data(self) -> None: + self.raw_dataset.save_to_disk(self.raw_paths[0]) + torch.save(self.split_idxs, self.raw_paths[1]) + + def _load_raw_data(self) -> None: + if not hasattr(self, "raw_dataset"): + self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) + if not hasattr(self, "split_idxs"): + self.split_idxs = torch.load(self.raw_paths[1]) def download(self) -> None: dataset = datasets.load_dataset("rmanluo/RoG-webqsp") @@ -199,58 +249,115 @@ def download(self) -> None: torch.arange(len(dataset["validation"])) + len(dataset["train"]), "test": torch.arange(len(dataset["test"])) + len(dataset["train"]) + - len(dataset["validation"]) + len(dataset["validation"]), } - def process(self) -> None: - self.model = SentenceTransformer( - "sentence-transformers/all-roberta-large-v1").to(self.device) - self.model.eval() - self.questions = [i["question"] for i in self.raw_dataset] - list_of_graphs = [] - # encode questions + if self.limit >= 0: + self.raw_dataset = self.raw_dataset.select(range(self.limit)) + self.split_idxs = { + "train": + torch.arange(self.limit // 2), + "val": + torch.arange(self.limit // 4) + self.limit // 2, + "test": + torch.arange(self.limit // 4) + self.limit // 2 + + self.limit // 4, + } + self._save_raw_data() + + def _get_trips(self) -> Iterator[TripletLike]: + return chain.from_iterable( + iter(ds["graph"]) for ds in self.raw_dataset) + + def _build_graph(self) -> None: + trips = self._get_trips() + self.indexer: LargeGraphIndexer = LargeGraphIndexer.from_triplets( + trips, pre_transform=preprocess_triplet) + + # Nodes: + nodes = self.indexer.get_unique_node_features() + x = self.model.encode(nodes, batch_size=256) # type: ignore + self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) + + # Edges: + edges = self.indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) + edge_attr = self.model.encode(edges, batch_size=256) # type: ignore + self.indexer.add_edge_feature( + new_feature_name="edge_attr", + new_feature_vals=edge_attr, + map_from_feature=EDGE_RELATION, + ) + + print("Saving graph...") + self.indexer.save(self.processed_paths[-1]) + + def _retrieve_subgraphs(self) -> None: print("Encoding questions...") + self.questions = [str(ds["question"]) for ds in self.raw_dataset] q_embs = self.model.encode(self.questions, batch_size=256) - print("Encoding graphs...") + list_of_graphs = [] + print("Retrieving subgraphs...") + textual_nodes = self.textual_nodes + textual_edges = self.textual_edges + if self.whole_graph_retrieval: + graph = self.indexer.to_data(node_feature_name="x", + edge_feature_name="edge_attr") + else: + graph_gen = get_features_for_triplets_groups( + self.indexer, (ds['graph'] for ds in self.raw_dataset), + pre_transform=preprocess_triplet) + for index in tqdm(range(len(self.raw_dataset))): data_i = self.raw_dataset[index] - raw_nodes: Dict[str, int] = {} - raw_edges = [] - for tri in data_i["graph"]: - h, r, t = tri - h = h.lower() - t = t.lower() - if h not in raw_nodes: - raw_nodes[h] = len(raw_nodes) - if t not in raw_nodes: - raw_nodes[t] = len(raw_nodes) - raw_edges.append({ - "src": raw_nodes[h], - "edge_attr": r, - "dst": raw_nodes[t] - }) - nodes = DataFrame([{ - "node_id": v, - "node_attr": k - } for k, v in raw_nodes.items()], columns=["node_id", "node_attr"]) - edges = DataFrame(raw_edges, columns=["src", "edge_attr", "dst"]) - # encode nodes - nodes.node_attr = nodes.node_attr.fillna("") - x = self.model.encode(nodes.node_attr.tolist(), batch_size=256) - # encode edges - edge_attr = self.model.encode(edges.edge_attr.tolist(), - batch_size=256) - edge_index = torch.LongTensor( - [edges.src.tolist(), edges.dst.tolist()]) + if not self.whole_graph_retrieval: + graph = next(graph_gen) + textual_nodes = self.textual_nodes.iloc[ + graph["node_idx"]].reset_index() + textual_edges = self.textual_edges.iloc[ + graph["edge_idx"]].reset_index() + pcst_subgraph, desc = retrieval_via_pcst( + graph, + q_embs[index], + textual_nodes, + textual_edges, + topk=3, + topk_e=5, + cost_e=0.5, + override=not self.include_pcst, + ) question = f"Question: {data_i['question']}\nAnswer: " label = ("|").join(data_i["answer"]).lower() - raw_graph = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, - num_nodes=len(nodes)).to("cpu") - pcst_subgraph, desc = retrieval_via_pcst(raw_graph, q_embs[index], - nodes, edges, topk=3, - topk_e=5, cost_e=0.5) + pcst_subgraph["question"] = question pcst_subgraph["label"] = label pcst_subgraph["desc"] = desc list_of_graphs.append(pcst_subgraph.to("cpu")) + print("Saving subgraphs...") self.save(list_of_graphs, self.processed_paths[0]) + + def process(self) -> None: + self._load_raw_data() + self.model = SentenceTransformer( + 'sentence-transformers/all-roberta-large-v1').to(self.device) + self.model.eval() + if not os.path.exists(self.processed_paths[-1]): + print("Encoding graph...") + self._build_graph() + else: + print("Loading graph...") + self.indexer = LargeGraphIndexer.from_disk( + self.processed_paths[-1]) + self.textual_nodes = DataFrame.from_dict( + {"node_attr": self.indexer.get_node_features()}) + self.textual_nodes["node_id"] = self.textual_nodes.index + self.textual_nodes = self.textual_nodes[["node_id", "node_attr"]] + self.textual_edges = DataFrame(self.indexer.get_edge_features(), + columns=["src", "edge_attr", "dst"]) + self.textual_edges["src"] = [ + self.indexer._nodes[h] for h in self.textual_edges["src"] + ] + self.textual_edges["dst"] = [ + self.indexer._nodes[h] for h in self.textual_edges["dst"] + ] + self._retrieve_subgraphs() From e660a9bcfcd8907308a3490efd977f0c06a33f91 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 15:56:05 -0700 Subject: [PATCH 687/752] rename --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 03204960e3a7..fae3617e5b01 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -165,7 +165,7 @@ def preprocess_triplet(triplet: TripletLike) -> TripletLike: return str(h).lower(), str(r), str(t).lower() -class UpdatedWebQSPDataset(InMemoryDataset): +class WebQSPDataset(InMemoryDataset): r"""The WebQuestionsSP dataset was released as part of “The Value of Semantic Parse Labeling for Knowledge Base Question Answering” From 759f0f89b7fd5d17e9cbb35a5479a7fb31a33b39 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 16:00:17 -0700 Subject: [PATCH 688/752] merger 2 --- examples/llm_plus_gnn/multihop/rag_generate_multihop.py | 2 +- .../llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py | 4 ++-- examples/llm_plus_gnn/rag_generate.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index c4e0f6ad7544..83d3f9ec44b7 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -6,7 +6,7 @@ from rag_feature_store import SentenceTransformerApproxFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.datasets.updated_web_qsp_dataset import ( +from torch_geometric.datasets.web_qsp_dataset import ( preprocess_triplet, retrieval_via_pcst, ) diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index a615ea0d17b0..77ff8d214c28 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -5,7 +5,7 @@ from torch_geometric.loader import rag_loader from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet +from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet from torch_geometric.data import get_features_for_triplets_groups, Data from itertools import chain from torch_geometric.profile.nvtx import nvtxit @@ -43,7 +43,7 @@ fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() # %% -from torch_geometric.datasets.updated_web_qsp_dataset import retrieval_via_pcst +from torch_geometric.datasets.web_qsp_dataset import retrieval_via_pcst @nvtxit() def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index e0b5f2b546d0..b7b78438ab14 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -5,7 +5,7 @@ from torch_geometric.loader import RAGQueryLoader from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst +from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet, retrieval_via_pcst from torch_geometric.data import get_features_for_triplets_groups, Data from itertools import chain import torch From e5668bdf2c032cd28afadd72d87c00c87a062962 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 16:34:34 -0700 Subject: [PATCH 689/752] verbose --- torch_geometric/data/large_graph_indexer.py | 10 ++++++--- torch_geometric/datasets/web_qsp_dataset.py | 25 +++++++++------------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index e1f7b594ca9f..92f5cd492e78 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -565,7 +565,8 @@ def get_features_for_triplets_groups( indexer: LargeGraphIndexer, triplet_groups: Iterable[KnowledgeGraphLike], node_feature_name: str = "x", edge_feature_name: str = "edge_attr", - pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None + pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, + verbose: bool = False, ) -> Iterator[Data]: """Given an indexer and a series of triplet groups (like a dataset), retrieve the specified node and edge features for each triplet from the @@ -582,6 +583,7 @@ def get_features_for_triplets_groups( pre_transform (Optional[Callable[[TripletLike], TripletLike]]): Optional preprocessing to perform on triplets. Defaults to None. + verbose (bool, optional): Whether to print progress. Defaults to False. Yields: Iterator[Data]: For each triplet group, yield a data object containing @@ -601,7 +603,7 @@ def apply_transform(trips): edge_keys = [] edge_index = [] - for triplets in tqdm(triplet_groups): + for triplets in tqdm(triplet_groups, disable=not verbose): small_graph_indexer = LargeGraphIndexer.from_triplets( triplets, pre_transform=pre_transform) @@ -642,6 +644,7 @@ def get_features_for_triplets( node_feature_name: str = "x", edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, + verbose: bool = False, ) -> Data: """For a given set of triplets retrieve a Data object containing the unique graph and features from the index. @@ -655,6 +658,7 @@ def get_features_for_triplets( Defaults to "edge_attr". pre_transform (Optional[Callable[[TripletLike], TripletLike]]): Optional preprocessing function for triplets. Defaults to None. + verbose (bool, optional): Whether to print progress. Defaults to False. Returns: Data: Data object containing the unique graph and features from the @@ -662,5 +666,5 @@ def get_features_for_triplets( """ gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, - edge_feature_name, pre_transform) + edge_feature_name, pre_transform, verbose) return next(gen) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index fae3617e5b01..699d6cd9c241 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -254,6 +254,8 @@ def download(self) -> None: if self.limit >= 0: self.raw_dataset = self.raw_dataset.select(range(self.limit)) + + # HACK self.split_idxs = { "train": torch.arange(self.limit // 2), @@ -300,22 +302,17 @@ def _retrieve_subgraphs(self) -> None: print("Retrieving subgraphs...") textual_nodes = self.textual_nodes textual_edges = self.textual_edges - if self.whole_graph_retrieval: - graph = self.indexer.to_data(node_feature_name="x", - edge_feature_name="edge_attr") - else: - graph_gen = get_features_for_triplets_groups( - self.indexer, (ds['graph'] for ds in self.raw_dataset), - pre_transform=preprocess_triplet) + graph_gen = get_features_for_triplets_groups( + self.indexer, (ds['graph'] for ds in self.raw_dataset), + pre_transform=preprocess_triplet) - for index in tqdm(range(len(self.raw_dataset))): + for index in tqdm(range(len(self.raw_dataset)), disable=True): data_i = self.raw_dataset[index] - if not self.whole_graph_retrieval: - graph = next(graph_gen) - textual_nodes = self.textual_nodes.iloc[ - graph["node_idx"]].reset_index() - textual_edges = self.textual_edges.iloc[ - graph["edge_idx"]].reset_index() + graph = next(graph_gen) + textual_nodes = self.textual_nodes.iloc[ + graph["node_idx"]].reset_index() + textual_edges = self.textual_edges.iloc[ + graph["edge_idx"]].reset_index() pcst_subgraph, desc = retrieval_via_pcst( graph, q_embs[index], From dc54508d5dfca033ead9ea6f74d7ef256bf1919c Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 16:35:31 -0700 Subject: [PATCH 690/752] out file --- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 2 +- examples/llm_plus_gnn/rag_generate.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb index 17778a50b125..99fc723921e3 100644 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb @@ -198,7 +198,7 @@ "source": [ "import torch\n", "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet" + "from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet" ] }, { diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index b7b78438ab14..f3303a97a5b8 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -19,7 +19,7 @@ # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") parser.add_argument("--num_samples", type=int, default=4700) -parser.add_argument("--out_file", default="subg_results.pt") +parser.add_argument("--out_file", default="subg_results") args = parser.parse_args() # %% @@ -98,6 +98,6 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subg['question'] = questions[i] subg['label'] = ds[i]['label'] -pd.DataFrame.from_dict(retrieval_stats).to_csv('metadata.csv') -torch.save(subgs, args.out_file) +pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file+'_metadata.csv') +torch.save(subgs, args.out_file+'.pt') From 0359b09b938f287b004bb34b2846c225b32caa5f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 18:02:59 -0700 Subject: [PATCH 691/752] rm separate file for benchmarking multitoken gnns and readd changes that allowed for multiple output tokens to gretriever # Conflicts: # examples/llm_plus_gnn/benchmark_model_archs.py --- .../llm_plus_gnn/benchmark_model_archs.py | 86 +++++++++++++++++++ test/nn/models/test_g_retriever.py | 45 ++++++++-- torch_geometric/nn/models/g_retriever.py | 5 +- 3 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 examples/llm_plus_gnn/benchmark_model_archs.py diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py new file mode 100644 index 000000000000..1e586b9f94b3 --- /dev/null +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -0,0 +1,86 @@ +"""Used to benchmark the performance of an untuned/fine tuned LLM against +GRetriever with various architectures and layer depths.""" +# %% +import argparse + +import torch +from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.nn.models import GAT, MLP, GRetriever + +from g_retriever import benchmark_models, get_loss, inference_step + +# %% +parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") +parser.add_argument("--hidden_channels", type=int, default=1024) +parser.add_argument("--learning_rate", type=float, default=1e-5) +parser.add_argument("--epochs", type=int, default=2) +parser.add_argument("--batch_size", type=int, default=8) +parser.add_argument("--eval_batch_size", type=int, default=16) +parser.add_argument("--tiny_llama", action='store_true') + +parser.add_argument("--dataset_path", type=str, required=False) +# Default to WebQSP split +parser.add_argument("--num_train", type=int, default=2826) +parser.add_argument("--num_val", type=int, default=246) +parser.add_argument("--num_test", type=int, default=1628) + +args = parser.parse_args() + +# %% +hidden_channels = args.hidden_channels +lr = args.learning_rate +epochs = args.epochs +batch_size = args.batch_size +eval_batch_size = args.eval_batch_size + +# %% +if not args.dataset_path: + ds = UpdatedWebQSPDataset('benchmark_archs') +else: + # We just assume that the size of the dataset accomodates the + # train/val/test split, because checking may be expensive. + dataset = torch.load(args.dataset_path) + + class MockDataset: + """Utility class to patch the fields in WebQSPDataset used by GRetriever.""" + def __init__(self) -> None: + pass + + @property + def split_idxs(self) -> dict: + # Imitates the WebQSP split method + return { + "train": torch.arange(args.num_train), + "val": torch.arange(args.num_val) + args.num_train, + "test": torch.arange(args.num_test) + args.num_train + args.num_val, + } + def __getitem__(self, idx: int): + return dataset[idx] + ds = MockDataset() + +# %% +model_names = [] +model_classes = [] +model_kwargs = [] +model_type = ["GAT", "MLP"] +models = {"GAT": GAT, "MLP": MLP} +num_layers = [4] # Use to vary the depth of the GNN model +num_tokens = [1] # Use to vary the number of LLM tokens reserved for GNN output +for m_type in model_type: + for n_layer in num_layers: + for n_tokens in num_tokens: + model_names.append(f"{m_type}_{n_layer}_{n_tokens}") + model_classes.append(GRetriever) + kwargs = dict(gnn_hidden_channels=hidden_channels, + num_gnn_layers=n_layer, gnn_to_use=models[m_type], mlp_out_tokens=n_tokens) + if args.tiny_llama: + kwargs['llm_to_use'] = 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' + kwargs['mlp_out_dim'] = 2048 + kwargs['num_llm_params'] = 1 + model_kwargs.append(kwargs) + +# %% +benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, + batch_size, eval_batch_size, get_loss, inference_step, + skip_LLMs=False, tiny_llama=args.tiny_llama, force=True) + diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 01be4313da47..e29247203851 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -5,9 +5,7 @@ from torch_geometric.testing import onlyFullTest, withPackage -@onlyFullTest -@withPackage('transformers', 'sentencepiece', 'accelerate') -def test_g_retriever() -> None: +def _setup_batch() -> Data: batch = Data() batch.question = ["Is PyG the best open-source GNN library?"] batch.label = ["yes!"] @@ -23,19 +21,48 @@ def test_g_retriever() -> None: # simple cycle graph batch.edge_index = torch.tensor( [list(range(10)), list(range(1, 10)) + [0]]) + return batch - model = GRetriever( - llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", - num_llm_params=1, # 1 Billion - mlp_out_dim=2048, - ) - batch = batch.to(model.llm_device) +def _eval_train_call(model, batch): # test train loss = model(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.label, batch.edge_attr) assert loss is not None +def _eval_inference_call(model, batch): # test inference pred = model.inference(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.edge_attr) assert pred is not None + +@onlyFullTest +@withPackage('transformers', 'sentencepiece', 'accelerate') +def test_g_retriever() -> None: + batch = _setup_batch() + + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + mlp_out_dim=2048, + ) + batch = batch.to(model.llm_device) + + _eval_train_call(model, batch) + _eval_inference_call(model, batch) + +@onlyFullTest +@withPackage('transformers', 'sentencepiece', 'accelerate') +def test_g_retriever_many_tokens() -> None: + batch = _setup_batch() + + model = GRetriever( + llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", + num_llm_params=1, # 1 Billion + mlp_out_dim=2048, + mlp_out_tokens=2 + ) + batch = batch.to(model.llm_device) + + _eval_train_call(model, batch) + _eval_inference_call(model, batch) + diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 0a9182bdf858..1d7462060b29 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -33,6 +33,8 @@ class GRetriever(nn.Module): :obj:`heads` kwarg. (default: :obj:`4`) mlp_hidden_dim (int): (default: :obj:`2048`) mlp_out_dim (int): (default: :obj:`4096`) + mlp_out_tokens (int) : Number of LLM prefix tokens to reserve for + GNN output. (default: :obj:`1`) .. warning:: This module has been tested with the following Hugging Face models @@ -117,6 +119,7 @@ def __init__( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), nn.Linear(mlp_hidden_dim, mlp_out_dim*mlp_out_tokens), + nn.Unflatten(-1, (mlp_out_tokens, mlp_out_dim)), ).to(self.llm_device) self.mlp_out_tokens = mlp_out_tokens @@ -223,7 +226,7 @@ def inference( graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, batch) graph_embeds = [ - (embed.unsqueeze(0) if num_nodes_per_graph[i] != 0 else None) + (embed if num_nodes_per_graph[i] != 0 else None) for i, embed in enumerate(self.projector(graph_embeds)) ] inputs_embeds, attention_mask, _ = self.llm_to_use._get_embeds( From e045ed407e540134ae7a0a1fc8036939062019aa Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 18:11:24 -0700 Subject: [PATCH 692/752] rm from benchmark section --- benchmark/llm_rag/benchmark_model_archs.py | 84 -------------------- benchmark/llm_rag/benchmark_model_gnn_enc.py | 42 ---------- 2 files changed, 126 deletions(-) delete mode 100644 benchmark/llm_rag/benchmark_model_archs.py delete mode 100644 benchmark/llm_rag/benchmark_model_gnn_enc.py diff --git a/benchmark/llm_rag/benchmark_model_archs.py b/benchmark/llm_rag/benchmark_model_archs.py deleted file mode 100644 index 9b92c74134cc..000000000000 --- a/benchmark/llm_rag/benchmark_model_archs.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Used to benchmark the performance of an untuned/fine tuned LLM against -GRetriever with various architectures and layer depths.""" -# %% -import argparse - -import torch -from torch_geometric.datasets import UpdatedWebQSPDataset -from torch_geometric.nn.models import GAT, MLP, GRetriever - -from g_retriever import benchmark_models, get_loss, inference_step - -# %% -parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") -parser.add_argument("--hidden_channels", type=int, default=1024) -parser.add_argument("--learning_rate", type=float, default=1e-5) -parser.add_argument("--epochs", type=int, default=2) -parser.add_argument("--batch_size", type=int, default=8) -parser.add_argument("--eval_batch_size", type=int, default=16) -parser.add_argument("--tiny_llama", action='store_true') - -parser.add_argument("--dataset_path", type=str, required=False) -# Default to WebQSP split -parser.add_argument("--num_train", type=int, default=2826) -parser.add_argument("--num_val", type=int, default=246) -parser.add_argument("--num_test", type=int, default=1628) - -args = parser.parse_args() - -# %% -hidden_channels = args.hidden_channels -lr = args.learning_rate -epochs = args.epochs -batch_size = args.batch_size -eval_batch_size = args.eval_batch_size - -# %% -if not args.dataset_path: - ds = UpdatedWebQSPDataset('benchmark_archs') -else: - # We just assume that the size of the dataset accomodates the - # train/val/test split, because checking may be expensive. - dataset = torch.load(args.dataset_path) - - class MockDataset: - """Utility class to patch the fields in WebQSPDataset used by GRetriever.""" - def __init__(self) -> None: - pass - - @property - def split_idxs(self) -> dict: - # Imitates the WebQSP split method - return { - "train": torch.arange(args.num_train), - "val": torch.arange(args.num_val) + args.num_train, - "test": torch.arange(args.num_test) + args.num_train + args.num_val, - } - def __getitem__(self, idx: int): - return dataset[idx] - ds = MockDataset() - -# %% -model_names = [] -model_classes = [] -model_kwargs = [] -model_type = ["GAT", "MLP"] -models = {"GAT": GAT, "MLP": MLP} -num_layers = [4] -for m_type in model_type: - for n_layer in num_layers: - model_names.append(f"{m_type}_{n_layer}") - model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels, - num_gnn_layers=n_layer, gnn_to_use=models[m_type]) - if args.tiny_llama: - kwargs['llm_to_use'] = 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' - kwargs['mlp_out_dim'] = 2048 - kwargs['num_llm_params'] = 1 - model_kwargs.append(kwargs) - -# %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, - batch_size, eval_batch_size, get_loss, inference_step, - skip_LLMs=False, tiny_llama=args.tiny_llama, force=True) - diff --git a/benchmark/llm_rag/benchmark_model_gnn_enc.py b/benchmark/llm_rag/benchmark_model_gnn_enc.py deleted file mode 100644 index d777d9f8dede..000000000000 --- a/benchmark/llm_rag/benchmark_model_gnn_enc.py +++ /dev/null @@ -1,42 +0,0 @@ -# %% -from torch_geometric.datasets import WebQSPDataset -from torch_geometric.nn.models import GAT, MLP, GRetriever - -from g_retriever import benchmark_models, get_loss, inference_step - -# %% -hidden_channels = 1024 -num_gnn_layers = 1 -lr = 1e-5 -epochs = 2 -batch_size = 8 -eval_batch_size = 16 - -# %% -ds = WebQSPDataset('benchmark_archs') - -# %% -model_names = [] -model_classes = [] -model_kwargs = [] -model_type = ["GAT"] -models = {"GAT": GAT, "MLP": MLP} -num_layers = [4] -num_tokens = [4, 16] -for m_type in model_type: - for n_layer in num_layers: - for n_tokens in num_tokens: - model_names.append(f"{m_type}_{n_layer}_{n_tokens}") - model_classes.append(GRetriever) - kwargs = dict(gnn_hidden_channels=hidden_channels * n_tokens, - num_gnn_layers=n_layer, gnn_to_use=models[m_type], - gnn_out_channels=hidden_channels * n_tokens, - mlp_out_tokens=n_tokens) - model_kwargs.append(kwargs) - -# %% -benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, - batch_size, eval_batch_size, get_loss, inference_step, - skip_LLMs=False, tiny_llama=True, force=True) - -# %% From 60bd11f5cf569374c76cff7e421303ed45166c2d Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 18:54:06 -0700 Subject: [PATCH 693/752] fix formatting --- examples/llm_plus_gnn/benchmark_model_archs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index 1e586b9f94b3..e3519fd00406 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -64,8 +64,10 @@ def __getitem__(self, idx: int): model_kwargs = [] model_type = ["GAT", "MLP"] models = {"GAT": GAT, "MLP": MLP} -num_layers = [4] # Use to vary the depth of the GNN model -num_tokens = [1] # Use to vary the number of LLM tokens reserved for GNN output +# Use to vary the depth of the GNN model +num_layers = [4] +# Use to vary the number of LLM tokens reserved for GNN output +num_tokens = [1] for m_type in model_type: for n_layer in num_layers: for n_tokens in num_tokens: From e748cfcf51b85b08ab01b5dd72a756842a082d29 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 18:54:34 -0700 Subject: [PATCH 694/752] check dependencies for webqsp --- test/datasets/test_web_qsp_dataset.py | 11 ++++++++++- torch_geometric/datasets/web_qsp_dataset.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 45717997610f..53e8d2be54c2 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -10,7 +10,16 @@ def test_web_qsp_dataset(): assert str(dataset) == "WebQSPDataset(4700)" @onlyOnline +@onlyFullTest def test_web_qsp_dataset_limit(): dataset = WebQSPDataset(limit=100) assert len(dataset) == 100 - assert str(dataset) == "WebQSPDataset(100)" \ No newline at end of file + assert str(dataset) == "WebQSPDataset(100)" + +def test_web_qsp_import_error(): + # WebQSP shouldn't run if dependencies don't exist locally + try: + dataset = WebQSPDataset() + assert False # Test should fail if gets to here. + except ImportError: + pass \ No newline at end of file diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 699d6cd9c241..5d2518f448ad 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -198,6 +198,7 @@ def __init__( self.include_pcst = include_pcst self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") + self._check_dependencies() super().__init__(root, None, None, force_reload=force_reload) self._load_raw_data() self.load(self.processed_paths[0]) From 664cf7a9eff4a7d4fb9b42bbbe17b6f8d3eb4efd Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 18:59:08 -0700 Subject: [PATCH 695/752] multihop scripts --- .../multihop/multihop_download.sh | 32 +++ .../multihop/multihop_preprocess.py | 244 ++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100755 examples/llm_plus_gnn/multihop/multihop_download.sh create mode 100644 examples/llm_plus_gnn/multihop/multihop_preprocess.py diff --git a/examples/llm_plus_gnn/multihop/multihop_download.sh b/examples/llm_plus_gnn/multihop/multihop_download.sh new file mode 100755 index 000000000000..20517b83ac66 --- /dev/null +++ b/examples/llm_plus_gnn/multihop/multihop_download.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# Check if gzip is installed +if ! command -v gzip &> /dev/null +then + echo "gzip could not be found. Please install gzip and try again." + exit +fi + +# Check if wget is installed +if ! command -v wget &> /dev/null +then + echo "wget could not be found. Please install wget and try again." + exit +fi + +# Check if unzip is installed +if ! command -v unzip &> /dev/null +then + echo "unzip could not be found. Please install unzip and try again." + exit +fi + +# Wikidata5m +wget -O "https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz" +tar -xvf "wikidata5m_alias.tar.gz" +wget -O "https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz" +gzip -d "wikidata5m_all_triplet.txt.gz" -f + +# 2Multihopqa +wget -O "https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip" +unzip -o "data_ids_april7.zip" \ No newline at end of file diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop/multihop_preprocess.py new file mode 100644 index 000000000000..bc5f05a48582 --- /dev/null +++ b/examples/llm_plus_gnn/multihop/multihop_preprocess.py @@ -0,0 +1,244 @@ +"""Example workflow for downloading and assembling a multihop QA dataset""" + +# %% [markdown] +# # Encoding A Large Knowledge Graph Part 2 + +# %% [markdown] +# In this notebook, we will continue where we left off by building a new multi-hop QA dataset based on Wikidata. + +# %% [markdown] +# ## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph + +# %% [markdown] +# ### Motivation + +# %% [markdown] +# One potential application of knowledge graph structural encodings is capturing the relationships between different entities that are multiple hops apart. This can be challenging for an LLM to recognize from prepended graph information. Here's a motivating example (credit to @Rishi Puri): + +# %% [markdown] +# In this example, the question can only be answered by reasoning about the relationships between the entities in the knowledge graph. + +# %% [markdown] +# ### Building a Multi-Hop QA Dataset + +# %% [markdown] +# To start, we need to download the raw data of a knowledge graph. In this case, we use WikiData5M ([Wang et al](https://paperswithcode.com/paper/kepler-a-unified-model-for-knowledge)). Here we download the raw triplets and their entity codes. Information about this dataset can be found [here](https://deepgraphlearning.github.io/project/wikidata5m). + +# %% [markdown] +# The following download contains the ID to plaintext mapping for all the entities and relations in the knowledge graph: + +from subprocess import call +rv = call("./multihop_download.sh") + +# %% [markdown] +# To start, we are going to preprocess the knowledge graph to substitute each of the entity/relation codes with their plaintext aliases. This makes it easier to use a pre-trained textual encoding model to create triplet embeddings, as such a model likely won't understand how to properly embed the entity codes. + +# %% +import pandas as pd +import tqdm +import json + +# %% +# Substitute entity codes with their aliases +# Picking the first alias for each entity (rather arbitrarily) +alias_map = {} +rel_alias_map = {} +for line in open('wikidata5m_entity.txt'): + parts = line.strip().split('\t') + entity_id = parts[0] + aliases = parts[1:] + alias_map[entity_id] = aliases[0] +for line in open('wikidata5m_relation.txt'): + parts = line.strip().split('\t') + relation_id = parts[0] + relation_name = parts[1] + rel_alias_map[relation_id] = relation_name + +# %% +full_graph = [] +missing_total = 0 +total = 0 +for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')): + src, rel, dst = line.strip().split('\t') + if src not in alias_map: + missing_total += 1 + if dst not in alias_map: + missing_total += 1 + if rel not in rel_alias_map: + missing_total += 1 + total += 3 + full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)]) +print(f"Missing aliases: {missing_total}/{total}") + +# %% [markdown] +# Now `full_graph` represents the knowledge graph triplets in understandable plaintext. + +# %% [markdown] +# Next, we need a set of multi-hop questions that the Knowledge Graph will provide us with context for. We utilize a subset of [HotPotQA](https://hotpotqa.github.io/) ([Yang et. al.](https://arxiv.org/pdf/1809.09600)) called [2WikiMultiHopQA](https://github.com/Alab-NII/2wikimultihop) ([Ho et. al.](https://aclanthology.org/2020.coling-main.580.pdf)), which includes a subgraph of entities that serve as the ground truth justification for answering each multi-hop question: + +# %% +with open('train.json') as f: + train_data = json.load(f) +train_df = pd.DataFrame(train_data) +train_df['split_type'] = 'train' + +with open('dev.json') as f: + dev_data = json.load(f) +dev_df = pd.DataFrame(dev_data) +dev_df['split_type'] = 'dev' + +with open('test.json') as f: + test_data = json.load(f) +test_df = pd.DataFrame(test_data) +test_df['split_type'] = 'test' + +df = pd.concat([train_df, dev_df, test_df]) + +# %% +df.head() + +# %% +df['split_type'].value_counts() + +# %% +df['type'].value_counts() + +# %% [markdown] +# Now we need to extract the subgraphs + +# %% +df['graph_size'] = df['evidences_id'].apply(lambda row: len(row)) + +# %% +df['graph_size'].value_counts() + +# %% [markdown] +# (Optional) We take only questions where the evidence graph is greater than 0. (Note: this gets rid of the test set): + +# %% +# df = df[df['graph_size'] > 0] + +# %% +refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']] + +# %% [markdown] +# Checkpoint: + +# %% +refined_df.to_csv('wikimultihopqa_refined.csv', index=False) + +# %% [markdown] +# Now we need to check that all the entities mentioned in the question/answer set are also present in the Wikidata graph: + +# %% +relation_map = {} +with open('wikidata5m_relation.txt') as f: + for line in tqdm.tqdm(f): + parts = line.strip().split('\t') + for i in range(1, len(parts)): + if parts[i] not in relation_map: + relation_map[parts[i]] = [] + relation_map[parts[i]].append(parts[0]) + +# %% +entity_set = set() +with open('wikidata5m_entity.txt') as f: + for line in tqdm.tqdm(f): + entity_set.add(line.strip().split('\t')[0]) + +# %% +missing_entities = set() +missing_entity_idx = set() +for i, row in enumerate(refined_df.itertuples()): + for trip in row.evidences_id: + entities = trip[0], trip[2] + for entity in entities: + if entity not in entity_set: + # print(f'The following entity was not found in the KG: {entity}') + missing_entities.add(entity) + missing_entity_idx.add(i) + +# %% [markdown] +# Right now, we drop the missing entity entries. Additional preprocessing can be done here to resolve the entity/relation collisions, but that is out of the scope for this notebook. + +# %% +# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped. +refined_df.reset_index(inplace=True, drop=True) + +# %% +cleaned_df = refined_df.drop(missing_entity_idx) + +# %% [markdown] +# Now we save the resulting graph and questions/answers dataset: + +# %% +cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False) + +# %% +import torch + +# %% +torch.save(full_graph, 'wikimultihopqa_full_graph.pt') + +# %% [markdown] +# ### Question: How do we extract a contextual subgraph for a given query? + +# %% [markdown] +# The chosen retrieval algorithm is a critical component in the pipeline for affecting RAG performance. In the next section (1), we will demonstrate a naive method of retrieval for a large knowledge graph, and how to apply it to this dataset along with WebQSP. + +# %% [markdown] +# ### Preparing a Textualized Graph for LLM + +# %% [markdown] +# For now however, we need to prepare the graph data to be used as a plaintext prefix to the LLM. In order to do this, we want to prompt the LLM to use the unique nodes, and unique edge triplets of a given subgraph. In order to do this, we prepare a unique indexed node df and edge df for the knowledge graph now. This process occurs trivially with the LargeGraphIndexer: + +# %% +from torch_geometric.data import LargeGraphIndexer + +# %% +indexer = LargeGraphIndexer.from_triplets(full_graph) + +# %% +# Node DF +textual_nodes = pd.DataFrame.from_dict( + {"node_attr": indexer.get_node_features()}) +textual_nodes["node_id"] = textual_nodes.index +textual_nodes = textual_nodes[["node_id", "node_attr"]] + +# %% +textual_nodes.head() + +# %% [markdown] +# Notice how LargeGraphIndexer ensures that there are no duplicate indices: + +# %% +textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0] + +# %% +# Edge DF +textual_edges = pd.DataFrame(indexer.get_edge_features(), + columns=["src", "edge_attr", "dst"]) +textual_edges["src"] = [ + indexer._nodes[h] for h in textual_edges["src"] +] +textual_edges["dst"] = [ + indexer._nodes[h] for h in textual_edges["dst"] +] + +# %% [markdown] +# Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph. + +# %% +textual_edges.head() + +# %% [markdown] +# Now we can save the result + +# %% +textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False) +textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False) + +# %% [markdown] +# Now were done! This knowledge graph and dataset will get used later on in Section 1. + + From 74725391575396fa124076f3fa6699d8acb3817b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 19:35:20 -0700 Subject: [PATCH 696/752] adjust multihop --- .../multihop/rag_generate_multihop.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index 83d3f9ec44b7..10ee8ded00ce 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -12,13 +12,20 @@ ) from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer +import argparse + +# %% +parser = argparse.ArgumentParser(description="Generate new multihop dataset for rag") +# TODO: Add more arguments for configuring rag params +parser.add_argument("--num_samples", type=int) +args = parser.parse_args() # %% triplets = torch.load('wikimultihopqa_full_graph.pt') # %% df = pd.read_csv('wikimultihopqa_cleaned.csv') -questions = df['question_text'][:10] +questions = df['question_text'][:args.num_samples] # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -34,11 +41,26 @@ }, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerApproxFeatureStore).load() +# %% +from typing import Tuple +from torch_geometric.data import Data + +all_textual_nodes = pd.read_csv('wikimultihopqa_textual_nodes.csv') +all_textual_edges = pd.read_csv('wikimultihopqa_textual_edges.csv') + +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + q_emb = model.encode(query) + textual_nodes = all_textual_nodes.iloc[graph["node_idx"]].reset_index() + textual_edges = all_textual_edges.iloc[graph["edge_idx"]].reset_index() + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph["desc"] = desc + return out_graph + # %% query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40] * 3}, - local_filter=retrieval_via_pcst) + local_filter=apply_retrieval_via_pcst) # %% subgs = [] From 9f2e61872748b23bd560d193e016e0a82516400c Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 20:34:17 -0700 Subject: [PATCH 697/752] adjust multihop 2 --- .../multihop/multihop_download.sh | 30 +++------------ .../multihop/multihop_preprocess.py | 37 ++++++------------- .../multihop/rag_generate_multihop.py | 4 +- 3 files changed, 20 insertions(+), 51 deletions(-) diff --git a/examples/llm_plus_gnn/multihop/multihop_download.sh b/examples/llm_plus_gnn/multihop/multihop_download.sh index 20517b83ac66..3c1970d39440 100755 --- a/examples/llm_plus_gnn/multihop/multihop_download.sh +++ b/examples/llm_plus_gnn/multihop/multihop_download.sh @@ -1,32 +1,12 @@ #!/bin/sh -# Check if gzip is installed -if ! command -v gzip &> /dev/null -then - echo "gzip could not be found. Please install gzip and try again." - exit -fi - -# Check if wget is installed -if ! command -v wget &> /dev/null -then - echo "wget could not be found. Please install wget and try again." - exit -fi - -# Check if unzip is installed -if ! command -v unzip &> /dev/null -then - echo "unzip could not be found. Please install unzip and try again." - exit -fi - # Wikidata5m -wget -O "https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz" + +wget -O "wikidata5m_alias.tar.gz" "https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz" tar -xvf "wikidata5m_alias.tar.gz" -wget -O "https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz" +wget -O "wikidata5m_all_triplet.txt.gz" "https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz" gzip -d "wikidata5m_all_triplet.txt.gz" -f # 2Multihopqa -wget -O "https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip" -unzip -o "data_ids_april7.zip" \ No newline at end of file +wget -O "data_ids_april7.zip" "https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip" +unzip -o "data_ids_april7.zip" diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop/multihop_preprocess.py index bc5f05a48582..4272361d59da 100644 --- a/examples/llm_plus_gnn/multihop/multihop_preprocess.py +++ b/examples/llm_plus_gnn/multihop/multihop_preprocess.py @@ -37,6 +37,12 @@ import pandas as pd import tqdm import json +import argparse + +# %% +parser = argparse.ArgumentParser(description="Preprocess wikidata5m") +parser.add_argument("--n_triplets", type=int, default=-1) +args = parser.parse_args() # %% # Substitute entity codes with their aliases @@ -58,7 +64,12 @@ full_graph = [] missing_total = 0 total = 0 +limit = None if args.n_triplets == -1 else args.n_triplets +i = 0 + for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')): + if limit is not None and i >= limit: + break src, rel, dst = line.strip().split('\t') if src not in alias_map: missing_total += 1 @@ -68,6 +79,7 @@ missing_total += 1 total += 3 full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)]) + i += 1 print(f"Missing aliases: {missing_total}/{total}") # %% [markdown] @@ -94,24 +106,12 @@ df = pd.concat([train_df, dev_df, test_df]) -# %% -df.head() - -# %% -df['split_type'].value_counts() - -# %% -df['type'].value_counts() - # %% [markdown] # Now we need to extract the subgraphs # %% df['graph_size'] = df['evidences_id'].apply(lambda row: len(row)) -# %% -df['graph_size'].value_counts() - # %% [markdown] # (Optional) We take only questions where the evidence graph is greater than 0. (Note: this gets rid of the test set): @@ -205,15 +205,10 @@ textual_nodes["node_id"] = textual_nodes.index textual_nodes = textual_nodes[["node_id", "node_attr"]] -# %% -textual_nodes.head() # %% [markdown] # Notice how LargeGraphIndexer ensures that there are no duplicate indices: -# %% -textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0] - # %% # Edge DF textual_edges = pd.DataFrame(indexer.get_edge_features(), @@ -228,17 +223,9 @@ # %% [markdown] # Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph. -# %% -textual_edges.head() - # %% [markdown] # Now we can save the result # %% textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False) textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False) - -# %% [markdown] -# Now were done! This knowledge graph and dataset will get used later on in Section 1. - - diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index 10ee8ded00ce..5c2b37fa7dd1 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -2,6 +2,8 @@ import pandas as pd import torch import tqdm +import sys +sys.path.append('..') from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerApproxFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore @@ -25,7 +27,7 @@ # %% df = pd.read_csv('wikimultihopqa_cleaned.csv') -questions = df['question_text'][:args.num_samples] +questions = df['question'][:args.num_samples] # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") From 000931c20f9006a2f8672de99435b526f8965f47 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Wed, 28 Aug 2024 20:42:24 -0700 Subject: [PATCH 698/752] more tests for web qsp --- test/datasets/test_web_qsp_dataset.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 53e8d2be54c2..48d4fff8700e 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,7 +1,9 @@ from torch_geometric.datasets import WebQSPDataset from torch_geometric.testing import onlyFullTest, onlyOnline +import pytest +@pytest.mark.skip(reason="Times out") @onlyOnline @onlyFullTest def test_web_qsp_dataset(): @@ -16,6 +18,13 @@ def test_web_qsp_dataset_limit(): assert len(dataset) == 100 assert str(dataset) == "WebQSPDataset(100)" +@onlyOnline +@onlyFullTest +def test_web_qsp_dataset_limit_no_pcst(): + dataset = WebQSPDataset(limit=100, include_pcst=False) + assert len(dataset) == 100 + assert str(dataset) == "WebQSPDataset(100)" + def test_web_qsp_import_error(): # WebQSP shouldn't run if dependencies don't exist locally try: From db8e0047ec25557a576a2bd10996d2075ed36eff Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 01:40:30 -0700 Subject: [PATCH 699/752] fix unittests --- test/datasets/test_web_qsp_dataset.py | 8 -------- torch_geometric/nn/models/g_retriever.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 48d4fff8700e..247913c7db92 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -24,11 +24,3 @@ def test_web_qsp_dataset_limit_no_pcst(): dataset = WebQSPDataset(limit=100, include_pcst=False) assert len(dataset) == 100 assert str(dataset) == "WebQSPDataset(100)" - -def test_web_qsp_import_error(): - # WebQSP shouldn't run if dependencies don't exist locally - try: - dataset = WebQSPDataset() - assert False # Test should fail if gets to here. - except ImportError: - pass \ No newline at end of file diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 1d7462060b29..1f4c151e62c3 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -172,7 +172,7 @@ def forward( if num_nodes_in_graph_i == 0: graph_embeds.append(None) else: - graph_embeds.append(projected_graph_embeds[i].unsqueeze(0)) + graph_embeds.append(projected_graph_embeds[i]) ( inputs_embeds, From 99c8db5c54f039ca87adc7e122983063c5618906 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 05:38:20 -0700 Subject: [PATCH 700/752] rag docs part 2 --- docs/source/advanced/rag.rst | 2038 +--------------------------------- 1 file changed, 27 insertions(+), 2011 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 2dffedb3707e..06ce71b7d42d 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -58,17 +58,16 @@ are composed of the following steps: Encoding a Large Knowledge Graph ================================ -In this notebook, we are going to walk through how to encode a large -knowledge graph for the purposes of Graph RAG. We will provide two -examples of how to do so, along with demonstration code. +To start, a Large Knowledge Graph needs to be created from triplets or +multiple subgraphs in a dataset. Example 1: Building from Already Existing Datasets -------------------------------------------------- In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. -The same is true for GNN based RAG. Consider the following dataset -WebQSP: +The same is true for GNN based RAG. For example, consider the +WebQSPDataset. .. code:: ipython3 @@ -78,12 +77,6 @@ WebQSP: ds = WebQSPDataset('small_sample', limit=num_questions) -.. parsed-literal:: - - /home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html - from .autonotebook import tqdm as notebook_tqdm - - WebQSP is a dataset that is based off of a subset of the Freebase Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a @@ -125,1370 +118,17 @@ needed to answer the question. 'a_entity': ['Jaxon Bieber'], 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'], ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'], - ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'], - ['Rudolph Valentino', - 'freebase.valuenotation.is_reviewed', - 'Place of birth'], - ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'], - ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'], - ['Stephen Melton', 'people.person.nationality', 'United States of America'], - ['Record producer', - 'music.performance_role.regular_performances', - 'm.012m1vf1'], - ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'], - ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'], - ['2011 Teen Choice Awards', - 'award.award_ceremony.awards_presented', - 'm.0yrkr34'], - ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'], - ['As Long As You Love Me (Ferry Corsten radio)', - 'common.topic.notable_types', - 'Musical Recording'], - ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'], - ['Stratford', 'location.location.containedby', 'Canada'], - ['Singer', - 'base.lightweight.profession.specialization_of', - 'Musicians and Singers'], - ['Enrique Iglesias', 'people.person.profession', 'Singer'], - ['Beauty and a Beat (acoustic version)', - 'music.recording.artist', - 'Justin Bieber'], - ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'], - ['50 Cent', 'people.person.profession', 'Film Producer'], - ['As Long As You Love Me (Audien dubstep mix)', - 'music.recording.canonical_version', - 'As Long As You Love Me'], - ['Kevin Risto', 'people.person.gender', 'Male'], - ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'], - ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'], - ['Mary J. Blige', 'people.person.profession', 'Record producer'], - ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'], - ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'], - ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'], - ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'], - ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'], - ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'], - ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'], - ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'], - ['American Music Award for Favorite Pop/Rock Album', - 'award.award_category.winners', - 'm.0ndc259'], - ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'], - ['Ontario', 'location.administrative_division.country', 'Canada'], - ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'], - ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'], - ['Billboard Music Award for Top Streaming Artist', - 'award.award_category.winners', - 'm.0njhx1b'], - ['Justin Bieber', 'film.producer.film', "Justin Bieber's Believe"], - ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'], - ['Brandy Norwood', 'people.person.profession', 'Singer'], - ["Justin Bieber's Believe", 'film.film.personal_appearances', 'm.0101ft2j'], - ['Justin Bieber', 'music.artist.album', 'All Bad'], - ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'], - ['m.0v_729v', - 'tv.tv_guest_personal_appearance.episode', - 'Results Show: Week 7'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'], - ['One Less Lonely Girl', - 'music.album.primary_release', - 'One Less Lonely Girl'], - ['Twista', 'people.person.gender', 'Male'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'], - ['Ciara', 'broadcast.artist.content', 'FLOW 103'], - ['Jon M. Chu', 'film.director.film', "Justin Bieber's Believe"], - ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'], - ['Toby Gad', 'music.artist.genre', 'Rock music'], - ['Madonna', 'music.artist.genre', 'Pop music'], - ['Selena Gomez', 'music.artist.genre', 'Europop'], - ['m.0gbm3cg', - 'film.personal_film_appearance.film', - 'Justin Bieber: Never Say Never'], - ['Baby', 'music.recording.canonical_version', 'Baby'], - ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'], - ['Boyfriend', 'music.recording.artist', 'Justin Bieber'], - ['Dr. Dre', 'music.artist.genre', 'Rap music'], - ['MTV Video Music Award Japan for Best New Artist', - 'award.award_category.winners', - 'm.0yrhrwc'], - ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'], - ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'], - ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'], - ['m.0gctwjk', - 'tv.tv_guest_role.episodes_appeared_in', - 'Series 2, Episode 3'], - ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'], - ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'], - ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'], - ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'], - ['Selena Gomez', 'people.person.profession', 'Dancer'], - ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'], - ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'], - ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'], - ['As Long As You Love Me (PAULO & JACKINSKY radio)', - 'common.topic.notable_types', - 'Musical Recording'], - ['Beauty and a Beat', - 'music.single.versions', - 'Beauty and a Beat (Wideboys Club Mix)'], - ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Bryan Adams', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Madonna', 'people.person.profession', 'Singer-songwriter'], - ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'], - ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'], - ['Terence Dudley', 'music.artist.genre', 'Reggae'], - ['Kylie Minogue', 'people.person.profession', 'Actor'], - ['Adrienne Bailon', 'music.artist.genre', 'Pop music'], - ['Katy Perry', 'music.artist.genre', 'Electronic music'], - ['Dany Brillant', 'people.person.gender', 'Male'], - ['Martin Kierszenbaum', 'people.person.gender', 'Male'], - ['Anastacia', 'people.person.nationality', 'United States of America'], - ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'], - ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'], - ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Somebody to Love', 'music.composition.form', 'Song'], - ['Teen Choice Award for Choice Twitter Personality', - 'award.award_category.winners', - 'm.0yrkr34'], - ['Chef Tone', 'people.person.place_of_birth', 'Chicago'], - ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'], - ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'], - ['Record producer', - 'fictional_universe.character_occupation.characters_with_this_occupation', - 'Haley James Scott'], - ['Colbie Caillat', 'music.artist.genre', 'Pop music'], - ['C1', 'music.artist.genre', 'Contemporary R&B'], - ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'], - ['Kanye West', 'people.person.profession', 'Singer'], - ['Pop music', 'common.topic.subject_of', 'Stephen Melton'], - ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'], - ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'], - ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'], - ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'], - ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'], - ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'], - ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'], - ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'], - ['Live My Life', 'music.album.compositions', 'Live My Life'], - ['RedOne', 'music.artist.genre', 'Rock music'], - ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'], - ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'], - ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'], - ['Little Bird', 'common.topic.notable_types', 'Musical Recording'], - ['As Long As You Love Me (Ferry Corsten radio)', - 'music.recording.featured_artists', - 'Big Sean'], - ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'], - ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'], - ['Terius Nash', 'people.person.nationality', 'United States of America'], - ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'], - ['Athan Grace', 'people.person.profession', 'Actor'], - ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'], - ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'], - ['All Around the World (acoustic version)', - 'music.recording.artist', - 'Justin Bieber'], - ['Bad Day', 'music.composition.composer', 'Marvin Isley'], - ['Brandy Norwood', - 'influence.influence_node.influenced_by', - 'Whitney Houston'], - ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['MTV Video Music Award for Artist to Watch', - 'award.award_category.winners', - 'm.0n1ykxp'], - ['Caitlin Beadles', - 'celebrities.celebrity.sexual_relationships', - 'm.0d33gyj'], - ['As Long As You Love Me', - 'music.single.versions', - 'As Long As You Love Me (Audiobot instrumental)'], - ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'], - ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'], - ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'The Mighty Mighty Bosstones'], - ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'], - ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'], - ['Lolly', 'music.composition.composer', 'Juicy J'], - ['Justin Bieber: Never Say Never', - 'media_common.netflix_title.netflix_genres', - 'Documentary film'], - ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'], - ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Amerie', 'music.artist.genre', 'Pop music'], - ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'], - ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'], - ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'], - ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'], - ['As Long As You Love Me', - 'music.single.versions', - 'As Long As You Love Me (Audien dubstep edit)'], - ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'], - ['Recovery', 'music.recording.song', 'Recovery'], - ['Justin Timberlake', 'music.artist.genre', 'Electronic music'], - ['Mannie Fresh', 'people.person.nationality', 'United States of America'], - ['m.0101ftqp', 'film.film_crew_gig.film', "Justin Bieber's Believe"], - ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'], - ['Leif Garrett', 'music.artist.genre', 'Rock music'], - ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'], - ['First Dance', 'music.recording.artist', 'Justin Bieber'], - ['#thatPower', 'music.recording.song', '#thatPower'], - ['Children', 'rdf-schema#range', 'Person'], - ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'], - ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'], - ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'], - ['2013 Teen Choice Awards', - 'award.award_ceremony.awards_presented', - 'm.0wjgqck'], - ['The Island Def Jam Music Group', - 'organization.organization.parent', - 'm.04q65lb'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'Rusted Root'], - ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'], - ['m.0z87d3n', - 'award.award_honor.award', - 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'], - ['Shaffer Smith', 'music.artist.genre', 'Dance music'], - ['Live My Life', 'music.composition.composer', 'John Mamann'], - ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'], - ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'], - ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], - ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'], - ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'], - ['Tampa', 'location.location.containedby', 'United States of America'], - ['Love Never Felt So Good', - 'music.album.compositions', - 'Love Never Felt So Good'], - ['As Long As You Love Me (Ferry Corsten remix)', - 'music.recording.artist', - 'Justin Bieber'], - ['Nelly', 'music.artist.genre', 'Rhythm and blues'], - ['Marvin Isley', 'music.composer.compositions', 'Bad Day'], - ['Somebody to Love', 'common.topic.notable_types', 'Composition'], - ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'], - ['Snoop Dogg', 'people.person.gender', 'Male'], - ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'], - ['Estelle', 'people.person.profession', 'Record producer'], - ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'], - ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'], - ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'], - ['50 Cent', 'people.person.nationality', 'United States of America'], - ['Chris Jasper', 'people.person.gender', 'Male'], - ['Sir Nolan', 'music.artist.genre', 'Pop music'], - ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'], - ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'], - ['Snoop Dogg', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['David Nicksay', 'people.person.gender', 'Male'], - ['Justin Bieber', 'people.person.profession', 'Record producer'], - ['Everlast', 'people.person.profession', 'Singer-songwriter'], - ['Juno Awards of 2014', - 'award.award_ceremony.awards_presented', - 'm.0102z0vx'], - ['As Long As You Love Me (Audiobot remix)', - 'music.recording.song', - 'As Long as You Love Me'], - ['#thatPower', 'music.composition.composer', 'Will i Am'], - ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'], - ['m.0_cyzs_', - 'celebrities.legal_entanglement.offense', - 'Driving under the influence'], - ['LeAnn Rimes', 'people.person.profession', 'Actor'], - ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'], - ['Mann', 'people.person.gender', 'Male'], - ['JoJo', 'people.person.gender', 'Female'], - ['Right Here (featuring Drake)', - 'music.recording.canonical_version', - 'Right Here'], - ['Mason Levy', 'music.composer.compositions', 'Boyfriend'], - ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'], - ['m.0yrjynf', - 'award.award_honor.award', - 'Teen Choice Award for Choice Summer Music Star: Male'], - ['Pras', 'people.person.profession', 'Record producer'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'], - ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'], - ['My World 2.0', 'music.album.releases', 'My World 2.0'], - ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'], - ['Marvin Isley', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'], - ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'], - ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'], - ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'], - ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'], - ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['School Gyrls', 'film.film.language', 'English Language'], - ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'], - ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'], - ['Katy Perry', 'people.person.profession', 'Actor'], - ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'], - ['Ronald Isley', 'people.person.profession', 'Actor'], - ['Live My Life (Party Rock remix)', - 'music.recording.featured_artists', - 'Redfoo'], - ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'], - ['Jaxon Bieber', 'people.person.nationality', 'Canada'], - ['As Long as You Love Me (album version)', - 'common.topic.notable_types', - 'Musical Recording'], - ['Justin Bieber: Just Getting Started', - 'book.written_work.author', - 'Justin Bieber'], - ['BeirutNights.com Radio', - 'broadcast.content.artist', - 'Marc Maris vs. Ramone'], - ['Gwen Stefani', 'people.person.profession', 'Musician'], - ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'], - ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'], - ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'], - ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'], - ['Love Never Felt So Good', - 'music.album.releases', - 'Love Never Felt So Good'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'], - ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'], - ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)', - 'music.recording.canonical_version', - 'Beauty and a Beat'], - ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'], - ['Dance music', - 'broadcast.genre.content', - "PartyRadioUSA.net - Nation's #1 Party Authority"], - ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'], - ['Terius Nash', 'people.person.profession', 'Record producer'], - ['Terence Dudley', 'people.person.profession', 'Record producer'], - ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'], - ['Baby', 'common.topic.notable_types', 'Award-Winning Work'], - ['Lolly', 'music.recording.canonical_version', 'Lolly'], - ['Scooter Braun', 'people.person.gender', 'Male'], - ['Mistletoe', 'music.album.artist', 'Justin Bieber'], - ['Sir Nolan', 'people.person.gender', 'Male'], - ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'], - ["Justin Bieber's Believe", 'film.film.other_crew', 'm.0101ftt1'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'], - ['Synthpop', 'music.genre.parent_genre', 'K-pop'], - ['Adam Messinger', - 'music.composer.compositions', - "Turn to You (Mother's Day Dedication)"], - ['m.0yrktlv', - 'award.award_honor.award', - 'Teen Choice Award for Choice Male Hottie'], - ['Kanye West', 'people.person.nationality', 'United States of America'], - ['Iggy Azalea', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv4c'], - ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'], - ['JellyRadio.com', 'broadcast.content.artist', 'DMX'], - ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'], - ['m.0gxnnzy', - 'celebrities.romantic_relationship.relationship_type', - 'Dated'], - ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'], - ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'], - ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'], - ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fvcp'], - ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'], - ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Britney Spears', - 'broadcast.artist.content', - "PartyRadioUSA.net - Nation's #1 Party Authority"], - ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'], - ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'], - ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'], - ['Ludacris', 'music.artist.contribution', 'm.0vp800w'], - ['Singer', 'common.topic.subject_of', 'Justin Bieber'], - ['Fergie', 'music.artist.genre', 'Rock music'], - ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'], - ['Toby Gad', 'people.person.profession', 'Record producer'], - ['All Around The World', 'music.composition.composer', 'Justin Bieber'], - ['Mistletoe', 'music.album.release_type', 'Single'], - ['Kid Cudi', 'people.person.profession', 'Film Producer'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'], - ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'], - ['Live My Life (Party Rock remix)', - 'music.recording.tracks', - 'Live My Life (Party Rock remix)'], - ['Beauty and a Beat (Bisbetic Instrumental)', - 'music.recording.artist', - 'Justin Bieber'], - ['m.0njw4z2', - 'award.award_honor.award', - 'MTV Europe Music Award for Best Male'], - ["Destiny's Child", 'music.artist.genre', 'Contemporary R&B'], - ['Snoop Dogg', 'people.person.profession', 'Record producer'], - ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'], - ['m.0gbm3c3', - 'film.personal_film_appearance.type_of_appearance', - 'Him/Herself'], - ['Rodney Jerkins', 'people.person.nationality', 'United States of America'], - ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'], - ["PartyRadioUSA.net - Nation's #1 Party Authority", - 'broadcast.content.artist', - 'Miley Cyrus'], - ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'], - ["Destiny's Child", 'music.artist.genre', 'Pop music'], - ['United States of America', - 'base.biblioness.bibs_topic.is_really', - 'United States of America'], - ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['m.09xx941', 'common.webpage.topic', 'Teen idol'], - ['Christina Milian', 'people.person.profession', 'Record producer'], - ['JoJo', 'people.person.nationality', 'United States of America'], - ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'], - ['Next to You', 'music.album.release_type', 'Single'], - ['#thatPower', 'music.composition.recordings', '#thatPOWER'], - ['Willa Ford', 'people.person.languages', 'English Language'], - ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['All That Matters', 'music.composition.composer', 'Andre Harris'], - ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'], - ['Paul Anka', 'music.artist.genre', 'Pop music'], - ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], - ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'], - ['Caitlin Beadles', 'people.person.nationality', 'Canada'], - ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'], - ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'], - ['Hot Wired Radio', - 'broadcast.content.broadcast', - 'Hot Wired Radio - 128kbps Stream'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'], - ['Avery', 'common.topic.notable_types', 'Musical Artist'], - ['m.0gbm3d9', - 'film.personal_film_appearance.film', - 'Justin Bieber: Never Say Never'], - ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Beyoncé Knowles', 'people.person.profession', 'Actor'], - ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Tupac Shakur', 'people.person.profession', 'Actor'], - ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'], - ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'], - ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'], - ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Frank Ocean', 'people.person.nationality', 'United States of America'], - ['Christina Milian', 'music.composer.compositions', 'Baby'], - ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'], - ['Justin Timberlake', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Khalil', 'people.person.gender', 'Male'], - ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'], - ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'], - ['Selena Gomez', - 'freebase.valuenotation.has_no_value', - 'Spouse (or domestic partner)'], - ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'], - ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], - ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'], - ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'], - ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv7x'], - ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'], - ['Anastacia', 'music.artist.genre', 'Contemporary R&B'], - ['C1', 'music.artist.genre', 'Hip hop music'], - ['My Worlds Acoustic', - 'freebase.valuenotation.is_reviewed', - 'Album content type'], - ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'], - ['Live My Life', 'music.composition.language', 'English Language'], - ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'], - ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'], - ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'], - ['Baby', 'music.album.releases', 'Baby'], - ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'], - ['Right Here', 'music.recording.canonical_version', 'Right Here'], - ['Justin Bieber', 'people.person.profession', 'Musician'], - ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Bigger', 'music.composition.composer', 'Waynne Nugent'], - ['Home to Mama', 'music.composition.composer', 'Cody Simpson'], - ['Big R Radio - The Hawk', - 'broadcast.content.artist', - 'The Black Eyed Peas'], - ['Thought Of You', 'music.composition.composer', 'Justin Bieber'], - ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'], - ['Singer', 'people.profession.specializations', 'Prima donna'], - ['Alanis Morissette', 'people.person.profession', 'Record producer'], - ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'], - ['Record producer', 'common.topic.notable_for', 'g.1258k9617'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'], - ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'], - ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'], - ['Justin Bieber: Never Say Never', - 'film.film.production_companies', - 'AEG Live'], - ['Redfoo', 'people.person.gender', 'Male'], - ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'], - ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'], - ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'], - ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'], - ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'], - ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'], - ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Right Here', 'music.album.featured_artists', 'Drake'], - ['m.01053qzf', - 'film.personal_film_appearance.film', - 'Justin Bieber: Never Say Never'], - ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'], - ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'], - ['Jordan Pruitt', 'music.artist.genre', 'Pop music'], - ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'], - ['Thought of You', 'common.topic.notable_types', 'Canonical Version'], - ['Whitney Houston', 'people.person.profession', 'Record producer'], - ['m.07lkzw7', 'common.webpage.category', 'Official Website'], - ['Ray J', 'people.person.profession', 'Musician'], - ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'], - ['Enrique Iglesias', 'people.person.gender', 'Male'], - ['m.0101fv5f', - 'film.film_regional_release_date.film', - "Justin Bieber's Believe"], - ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'], - ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'], - ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'], - ['Selena Gomez', 'music.artist.genre', 'Teen pop'], - ["Justin Bieber's Believe", 'film.film.produced_by', 'Scooter Braun'], - ['Love Never Felt So Good', 'music.album.genre', 'Disco'], - ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'], - ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'], - ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['m.0v_729v', - 'tv.tv_guest_personal_appearance.appearance_type', - 'Guest host'], - ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'], - ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'], - ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'], - ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'], - ['All Around the World', 'music.recording.featured_artists', 'Ludacris'], - ['Christina Milian', 'people.person.profession', 'Actor'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'], - ['Dance music', 'broadcast.genre.content', '181-party'], - ['Queen Elizabeth II Diamond Jubilee Medal', - 'award.award_category.winners', - 'm.0njwqrb'], - ['Sean Kingston', 'people.person.profession', 'Singer'], - ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'], - ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'], - ['CMT Music Award: Collaborative Video of the Year', - 'award.award_category.winners', - 'm.0njvs9s'], - ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'], - ['One Time', 'common.topic.notable_types', 'Musical Album'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'], - ['Katy Perry', 'music.artist.genre', 'Disco'], - ['Chingy', 'people.person.profession', 'Actor'], - ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'], - ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'], - ['Rihanna', 'music.artist.genre', 'Dance-pop'], - ['Justin Bieber', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'], - ['Chingy', 'people.person.gender', 'Male'], - ['Reed Smoot', 'people.person.gender', 'Male'], - ["Justin Bieber's Believe", 'film.film.edited_by', 'Jillian Twigger Moul'], - ['Teyana', 'freebase.valuenotation.has_value', 'Parents'], - ['Next to You', 'music.recording.song', 'Next to You'], - ['All Bad', 'music.composition.composer', 'Jason \\"Poo Bear\\" Boyd'], - ['As Long as You Love Me', - 'music.album.releases', - 'As Long As You Love Me (remixes)'], - ['Teen Choice Award for Choice Music: Breakout Artist - Male', - 'award.award_category.winners', - 'm.0yrjvlh'], - ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'], - ['Singer', 'common.topic.article', 'm.09l6h'], - ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'], - ['Scooter Braun', 'film.producer.film', "Justin Bieber's Believe"], - ['Justin Bieber: Never Say Never', - 'award.award_winning_work.awards_won', - 'm.0pc670l'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'], - ['Beauty And A Beat', 'music.composition.form', 'Song'], - ['Britney Spears', 'music.artist.genre', 'Electronic dance music'], - ['HitzRadio.com', 'broadcast.content.artist', "Destiny's Child"], - ['Beyoncé Knowles', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Live My Life', 'music.recording.tracks', 'Live My Life'], - ['m.0njhyh_', - 'award.award_honor.award', - 'Billboard Music Award for Top Streaming Song (Video)'], - ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'], - ['Ludacris', 'people.person.nationality', 'United States of America'], - ['Justin Bieber: Never Say Never', - 'film.film.film_production_design_by', - 'Devorah Herbert'], - ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'], - ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'], - ['Drake', 'music.artist.genre', 'Rhythm and blues'], - ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'], - ['Nick Jonas', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'], - ['Lupe Fiasco', - 'broadcast.artist.content', - "PartyRadioUSA.net - Nation's #1 Party Authority"], - ['Martin Kierszenbaum', - 'people.person.place_of_birth', - 'United States of America'], - ['As Long as You Love Me', - 'music.composition.recordings', - 'As Long as You Love Me'], - ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'], - ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'], - ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'], - ['Beautiful', 'music.composition.lyricist', 'Toby Gad'], - ['Redfoo', 'music.artist.genre', 'Electronic dance music'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'], - ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'], - ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'], - ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'], - ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'], - ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'], - ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'], - ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], - ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'], - ['m.0y5tl39', - 'film.personal_film_appearance.film', - 'Les Coulisses des Golden Globes'], - ['Teen idol', 'common.topic.webpage', 'm.09y89l2'], - ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Kevin Risto', 'people.person.profession', 'Musician'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'], - ['justinbieber', - 'award.award_nominated_work.award_nominations', - 'm.0z0tgz6'], - ['Justin Bieber', 'music.artist.label', 'Island Records'], - ['Ernie Isley', 'people.person.nationality', 'United States of America'], - ['Kylie Minogue', 'people.person.profession', 'Film Producer'], - ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'], - ['Everlast', 'music.artist.label', 'Island Records'], - ['5th Annual Shorty Awards', - 'award.award_ceremony.awards_presented', - 'm.0ywvh8k'], - ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'], - ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'], - ['Baby', 'common.topic.notable_types', 'Composition'], - ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'], - ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'], - ['m.0tkqqgg', - 'award.award_nomination.award', - 'Juno Award for Pop Album of the Year'], - ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'], - ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'], - ['Person', 'type.type.properties', 'Parents'], - ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Nasri', 'people.person.profession', 'Singer'], - ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'], - ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'], - ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'], - ['As Long as You Love Me', - 'music.album.compositions', - 'As Long as You Love Me'], - ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'], - ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'], - ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'], - ['Bigger', 'music.composition.composer', 'Frank Ocean'], - ['Bigger', 'music.composition.recordings', 'Bigger'], - ['Canadian', 'common.topic.notable_types', 'Ethnicity'], - ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'], - ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'], - ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Chef Tone', 'people.person.nationality', 'United States of America'], - ['Whitney Houston', 'music.artist.genre', 'Dance music'], - ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'], - ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'], - ['Change Me', 'music.album.primary_release', 'Change Me'], - ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'], - ['m.0w3gbtv', - 'film.personal_film_appearance.film', - 'Zendaya: Behind the Scenes'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'], - ['That Should Be Me', 'music.composition.form', 'Song'], - ['Never Say Never', 'music.album.compositions', 'Never Say Never'], - ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'Justin Bieber'], - ['#thatPOWER', 'music.album.releases', '#thatPOWER'], - ['Ashley Tisdale', 'people.person.profession', 'Actor'], - ['Sir Nolan', 'music.artist.genre', 'Rock music'], - ['Beauty and a Beat (acoustic version)', - 'music.recording.song', - 'Beauty And A Beat'], - ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'], - ['Sia Furler', 'people.person.profession', 'Singer-songwriter'], - ['Usher', 'music.composer.compositions', 'First Dance'], - ['m.0n1ykxp', - 'award.award_honor.award', - 'MTV Video Music Award for Artist to Watch'], - ['Justin Bieber: Never Say Never', - 'media_common.netflix_title.netflix_genres', - 'Rockumentary'], - ['Amerie', 'people.person.gender', 'Female'], - ['Real Change: Artists for Education', - 'film.film.personal_appearances', - 'm.0y5th3r'], - ['Mistletoe', 'music.album.primary_release', 'Mistletoe'], - ['Beautiful and the Beat', - 'music.recording.canonical_version', - 'Beauty and a Beat'], - ['#Thatpower', 'music.recording.tracks', '#thatPOWER'], - ['Baby', 'common.topic.notable_types', 'Musical Album'], - ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'], - ['PYD', 'common.topic.notable_types', 'Composition'], - ['Ashlee Simpson', 'people.person.profession', 'Singer'], - ['Pray', 'music.album.artist', 'Justin Bieber'], - ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'], - ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'], - ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'], - ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'], - ['Geri Halliwell', 'people.person.profession', 'Model'], - ['iJustine', 'people.person.gender', 'Female'], - ['Nelly Furtado', 'people.person.gender', 'Female'], - ['Trey Songz', 'people.person.nationality', 'United States of America'], - ['m.0ng_vkd', - 'film.personal_film_appearance.film', - 'A Michael Bublé Christmas'], - ["Justin Bieber's Believe", 'film.film.produced_by', "Bill O'Dowd"], - ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'], - ['Ludacris', 'music.composer.compositions', 'Baby'], - ['Terius Nash', 'music.featured_artist.recordings', 'Baby'], - ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'], - ['Vanessa Hudgens', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'], - ['Beyoncé Knowles', 'people.person.profession', 'Record producer'], - ['#thatPOWER', 'music.recording.tracks', '#thatPower'], - ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'], - ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'], - ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'], - ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'], - ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'], - ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Madonna', 'people.person.profession', 'Record producer'], - ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'], - ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'], - ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'], - ['As Long as You Love Me (acoustic version)', - 'music.recording.song', - 'As Long as You Love Me'], - ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'], - ['Blu Cantrell', 'people.person.gender', 'Female'], - ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'], - ['Rob Thomas', 'people.person.gender', 'Male'], - ['Singer', 'people.profession.specializations', 'Piano Singer'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'], - ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'], - ['m.0hvlt03', - 'film.film_film_distributor_relationship.film', - 'Justin Bieber: Never Say Never'], - ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fvq6'], - ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'], - ['justinbieber', - 'award.award_nominated_work.award_nominations', - 'm.0_srv2b'], - ['Terence Dudley', 'people.person.profession', 'Musician'], - ['Donna Summer', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['m.0101fszs', - 'film.personal_film_appearance.film', - "Justin Bieber's Believe"], - ['Alanis Morissette', - 'freebase.valuenotation.is_reviewed', - 'Official website'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'Jenna Andrews'], - ['FLOW 103', 'broadcast.content.artist', 'Cherish'], - ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'], - ['Next to You', 'music.recording.song', 'Next to You'], - ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'], - ['Ray J', 'people.person.nationality', 'United States of America'], - ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'], - ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'], - ['m.0w3gbtv', - 'film.personal_film_appearance.type_of_appearance', - 'Him/Herself'], - ['Montell Jordan', 'music.artist.genre', 'Hip hop music'], - ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Fabolous', 'broadcast.artist.content', 'PowerHitz'], - ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'], - ['Nelly Furtado', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'], - ['Record producer', 'common.topic.webpage', 'm.09ygb05'], - ['As Long As You Love Me (Ferry Corsten remix)', - 'music.recording.canonical_version', - 'As Long As You Love Me'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'], - ['m.0gbm3fj', - 'film.personal_film_appearance.type_of_appearance', - 'Him/Herself'], - ['Bryan-Michael Cox', - 'freebase.valuenotation.is_reviewed', - 'Place of birth'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'], - ['As Long As You Love Me', - 'music.single.versions', - 'As Long As You Love Me (Ferry Corsten club dub)'], - ['Iggy Azalea', 'music.artist.genre', 'Synthpop'], - ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'], - ['As Long As You Love Me (Ferry Corsten club dub)', - 'common.topic.notable_types', - 'Musical Recording'], - ['#thatPOWER', 'music.album.album_content_type', 'Studio album'], - ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Katy Perry', 'music.artist.genre', 'Electronic dance music'], - ['Kid Cudi', 'people.person.profession', 'Record producer'], - ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'], - ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'], - ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Jaden Smith', 'people.person.profession', 'Dancer'], - ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'], - ['Keyshia Cole', 'people.person.profession', 'Record producer'], - ['Guest host', - 'tv.non_character_role.tv_guest_personal_appearances', - 'm.0v_98y7'], - ['Person', 'type.type.properties', 'Spouse (or domestic partner)'], - ['Fall Out Boy', 'music.artist.origin', 'Chicago'], - ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'], - ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'], - ['FLOW 103', 'broadcast.content.artist', '50 Cent'], - ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'], - ['L.A. Reid', 'music.producer.releases_produced', 'My World'], - ['L.A. Reid', 'people.person.gender', 'Male'], - ['Jessie J', 'music.artist.genre', 'Hip hop music'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'], - ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'], - ['Beauty and a Beat (Bisbetic Radio Mix)', - 'music.recording.artist', - 'Justin Bieber'], - ['London', 'location.location.containedby', 'Ontario'], - ['Justin Bieber: Never Say Never', - 'film.film.film_set_decoration_by', - 'Lia Roldan'], - ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Chris Brown', 'music.composer.compositions', 'Next to You'], - ['Beautiful', 'music.recording.tracks', 'Beautiful'], - ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'], - ['Children', 'type.property.schema', 'Person'], - ['Change Me', 'music.album.releases', 'Change Me'], - ['RedOne', 'music.artist.label', 'Island Records'], - ['School Gyrls', 'film.film.starring', 'm.0jztshx'], - ['All Around the World', - 'music.recording.canonical_version', - 'All Around the World'], - ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'], - ['Teen Choice Award for Choice Twitter Personality', - 'award.award_category.winners', - 'm.0wjhc6c'], - ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'], - ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'], - ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'], - ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'], - ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'], - ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'], - ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'], - ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'], - ["PartyRadioUSA.net - Nation's #1 Party Authority", - 'broadcast.content.artist', - 'Lupe Fiasco'], - ['Hikaru Utada', 'music.artist.label', 'Island Records'], - ['Dr. Dre', 'people.person.profession', 'Record producer'], - ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'], - ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Justin Bieber', 'music.composer.compositions', 'Change Me'], - ['Right Here', 'common.topic.notable_types', 'Composition'], - ['Change Me', 'music.composition.composer', 'Jason \\"Poo Bear\\" Boyd'], - ['Beauty and a Beat (Wideboys Radio Mix)', - 'music.recording.canonical_version', - 'Beauty and a Beat'], - ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'], - ['#Thatpower', 'music.recording.artist', 'Will i Am'], - ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'], - ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'], - ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'], - ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'], - ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'], - ['m.0v90skf', - 'award.award_honor.award', - 'Billboard Music Award for Top Male Artist'], - ['Ludacris', 'people.person.profession', 'Actor'], - ['Heartbreaker', 'music.album.genre', 'Pop music'], - ['Cameo appearance', - 'tv.special_tv_performance_type.episode_performances', - 'm.0v1lwt2'], - ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'], - ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'], - ['Model', 'base.lightweight.profession.similar_professions', 'Actor'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'], - ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'], - ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'], - ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'One Chance'], - ['Never Let You Go', 'common.topic.notable_types', 'Composition'], - ['Live My Life', 'common.topic.article', 'm.0j4453y'], - ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'], - ["Justin Bieber's Believe", 'film.film.personal_appearances', 'm.0y5t8gm'], - ['Roller Coaster', - 'award.award_nominated_work.award_nominations', - 'm.0_x4zg3'], - ['Chris Brown', 'people.person.nationality', 'United States of America'], - ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'], - ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'], - ['Teen pop', 'common.topic.article', 'm.02ny8z'], - ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'], - ['Iggy Azalea', 'people.person.gender', 'Female'], - ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Adrienne Bailon', 'people.person.profession', 'Dancer'], - ['Hip hop music', 'broadcast.genre.content', '181-beat'], - ['m.0sgk_cw', - 'award.award_honor.award', - "Kids' Choice Award for Favorite Song"], - ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'], - ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'], - ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'], - ['MTV Video Music Brazil Award for Best International Artist', - 'award.award_category.winners', - 'm.0yrhhqv'], - ['Mariah Carey', 'music.artist.label', 'Island Records'], - ['Music', 'common.topic.subject_of', 'POPPMusic.net'], - ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'], - ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Favorite Girl', 'music.album.artist', 'Justin Bieber'], - ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'], - ['Britney Spears', 'people.person.profession', 'Singer'], - ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'], - ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'], - ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'], - ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Rock music', - 'base.webvideo.internet_video_genre.series', - 'Biscuithands, The Animated Musical'], - ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Chef Tone', 'music.artist.genre', 'Hip hop music'], - ['Rudolph Isley', 'people.person.gender', 'Male'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'Barry Weiss'], - ['Beauty and a Beat (Bisbetic Instrumental)', - 'common.topic.notable_types', - 'Musical Recording'], - ['MTV Europe Music Award for Best Male', - 'award.award_category.winners', - 'm.0z1scxk'], - ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'], - ['Will Smith', 'people.person.profession', 'Actor'], - ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'], - ['Will i Am', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Boyfriend', 'music.composition.recordings', 'Boyfriend'], - ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'], - ['Fabian', 'people.person.gender', 'Male'], - ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'], - ['Somebody to Love (remix)', - 'music.album.primary_release', - 'Somebody to Love (remix)'], - ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'], - ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'], - ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'], - ['#thatPOWER', 'music.single.versions', '#thatPOWER'], - ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'], - ['Spouse', 'type.property.expected_type', 'Person'], - ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'], - ['Baby', 'music.recording.artist', 'Ludacris'], - ['Rudolph Valentino', - 'people.person.nationality', - 'United States of America'], - ['Hit-Boy', 'music.artist.genre', 'Hip hop music'], - ['Judy Garland', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Kelly Clarkson', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['#thatPower', 'music.composition.recordings', '#Thatpower'], - ["Justin Bieber's Believe", - 'base.schemastaging.context_name.pronunciation', - 'm.011h9_22'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'], - ['Miley Cyrus', 'people.person.profession', 'Musician'], - ['Justin Timberlake', 'people.person.gender', 'Male'], - ['#Thatpower', 'music.recording.tracks', '#thatPOWER'], - ['m.0vp8rhw', - 'music.recording_contribution.album', - 'Beauty and a Beat (Remixes)'], - ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'], - ['Katy Perry: Part of Me', - 'common.topic.notable_types', - 'Award-Winning Work'], - ['m.0jsmvv5', - 'film.film_regional_release_date.film', - 'Justin Bieber: Never Say Never'], - ["Justin Bieber's Believe", 'common.topic.notable_for', 'g.1yj4hbf4k'], - ['My Worlds: The Collection', 'music.album.release_type', 'Album'], - ['All Around The World (featuring Ludacris)', - 'music.recording.artist', - 'Justin Bieber'], - ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'], - ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'], - ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'], - ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'], - ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'], - ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'], - ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'], - ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'], - ['School Boy Records', 'music.record_label.artist', 'Madison Beer'], - ["Justin Bieber's Believe", 'film.film.other_crew', 'm.0101ftl5'], - ['Musical Album', 'freebase.type_hints.included_types', 'Topic'], - ['As Long As You Love Me', - 'music.single.versions', - 'As Long As You Love Me (Audien dubstep mix)'], - ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'], - ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'], - ['Fergie', 'people.person.profession', 'Actor'], - ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Stuart Ford', 'people.person.profession', 'Film Producer'], - ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'], - ['Zac Efron', 'people.person.gender', 'Male'], - ['P!nk', 'music.artist.genre', 'Rock music'], - ['R. Kelly', 'people.person.profession', 'Film Producer'], - ['Gender', 'type.property.schema', 'Person'], - ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'], - ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'], - ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'], - ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'], - ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'], - ['Under the Mistletoe', - 'freebase.valuenotation.is_reviewed', - 'Initial release date'], - ['Live My Life', 'music.recording.tracks', 'Live My Life'], - ['The Island Def Jam Music Group', - 'music.record_label.artist', - 'Slick Rick'], - ['Amerie', 'music.artist.genre', 'Rock music'], - ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['m.0pbzq13', - 'film.performance.special_performance_type', - 'Cameo appearance'], - ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'], - ['Height', 'type.property.unit', 'Meter'], - ['Iggy Azalea', 'people.person.profession', 'Model'], - ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'], - ['Ray J', - 'freebase.valuenotation.has_no_value', - 'Spouse (or domestic partner)'], - ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'], - ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Boyfriend (acoustic version)', - 'music.recording.canonical_version', - 'Boyfriend'], - ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'], - ['Believe Tour', - 'music.concert_tour.album_or_release_supporting', - 'Believe'], - ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Believe Acoustic', 'music.album.release_type', 'Album'], - ['Diplo', 'freebase.valuenotation.has_value', 'Height'], - ['Hikaru Utada', 'music.artist.genre', 'Synthpop'], - ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'], - ['Frank Ocean', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['As Long As You Love Me (Audiobot instrumental)', - 'music.recording.song', - 'As Long as You Love Me'], - ['Elvis Presley', 'music.artist.genre', 'Pop music'], - ['Lady Gaga', 'music.artist.genre', 'Pop music'], - ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'], - ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'], - ['Usher', 'people.person.nationality', 'United States of America'], - ['Live My Life', 'music.composition.recordings', 'Live My Life'], - ['Kelis', 'music.artist.genre', 'Contemporary R&B'], - ["Justin Bieber's Believe", 'film.film.release_date_s', 'm.0101fv5f'], - ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Next to You', 'music.recording.tracks', 'Next to You'], - ['m.0gbm3b7', - 'film.personal_film_appearance.type_of_appearance', - 'Him/Herself'], - ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'], - ['Sheryl Crow', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'], - ['All That Matters', - 'music.composition.composer', - 'Jason \\"Poo Bear\\" Boyd'], - ['Nasri', 'music.artist.genre', 'Reggae'], - ['#thatPOWER', 'music.recording.song', '#thatPower'], - ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'], - ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'], - ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'], - ['Bad 25', 'film.film.genre', 'Documentary film'], - ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'], - ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'], - ['As Long as You Love Me', - 'music.composition.recordings', - 'As Long As You Love Me'], - ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'], - ['Geri Halliwell', 'people.person.profession', 'Musician'], - ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'], - ['Bryan-Michael Cox', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'], - ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'], - ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'], - ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'], - ['Coldplay', 'music.artist.genre', 'Rock music'], - ['Kevin Risto', 'people.person.profession', 'Record producer'], - ['Whitney Houston', 'people.person.profession', 'Model'], - ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'], - ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'], - ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'], - ['As Long as You Love Me', - 'music.recording.canonical_version', - 'As Long As You Love Me'], - ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Disney Parks Christmas Day Parade', - 'common.topic.notable_types', - 'Award-Winning Work'], - ['Ray J', 'people.person.profession', 'Artist'], - ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'], - ['American Music Award for Favorite Pop/Rock Male Artist', - 'award.award_category.winners', - 'm.0ndc0sf'], - ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'], - ['The Notorious B.I.G.', - 'freebase.valuenotation.is_reviewed', - 'Place of birth'], - ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'], - ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Christina Aguilera', 'music.artist.genre', 'Electronic music'], - ['PowerHitz', 'broadcast.content.artist', 'Outkast'], - ['U Smile', 'music.music_video.artist', 'Justin Bieber'], - ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'], - ['Sean Kingston', 'music.artist.genre', 'Hip hop music'], - ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['Haley James Scott', - 'fictional_universe.fictional_character.occupation', - 'Record producer'], - ['Kylie Minogue', 'music.artist.genre', 'Rock music'], - ['Chris Jasper', 'people.person.nationality', 'United States of America'], - ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['My Worlds: The Collection', - 'music.album.album_content_type', - 'Compilation album'], - ['Lolly', 'music.album.releases', 'Lolly'], - ['Toby Gad', 'common.topic.notable_types', 'Record Producer'], - ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'], - ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'], - ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'], - ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'], - ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'], - ['Ja Rule', 'people.person.profession', 'Singer'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'], - ['Die in Your Arms', - 'award.award_nominated_work.award_nominations', - 'm.0z85qxq'], - ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'], - ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'], - ['Kuk Harrell', - 'film.person_or_entity_appearing_in_film.films', - 'm.0101ft5f'], - ['Somebody to Love (J Stax remix)', - 'music.recording.artist', - 'Justin Bieber'], - ["Justin Bieber's Believe", - 'film.film.executive_produced_by', - 'Allison Kaye Scarinzi'], - ['Adam Messinger', 'people.person.nationality', 'Canada'], - ['Nasri', 'music.artist.genre', 'Pop music'], - ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'], - ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'], - ['C1', 'common.topic.notable_types', 'Musical Artist'], - ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'], - ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'], - ['Country', 'freebase.type_profile.strict_included_types', 'Topic'], - ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'], - ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'], - ['m.05sp405', - 'organization.organization_relationship.child', - 'Island Records'], - ['Savan Kotecha', 'people.person.profession', 'Record producer'], - ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'], - ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'], - ['John Mamann', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['Teen Choice Award for Choice Summer Music Star: Male', - 'award.award_category.winners', - 'm.0yrjynf'], - ['Juicy J', 'people.person.profession', 'Actor'], - ['m.0yqflrk', - 'measurement_unit.dated_money_value.source', - 'celebritynetworth.com'], - ['Miley Cyrus', - 'freebase.valuenotation.is_reviewed', - 'Country of nationality'], - ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'], - ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'], - ['m.04q65lb', - 'organization.organization_relationship.child', - 'The Island Def Jam Music Group'], - ['Big Sean', 'people.person.nationality', 'United States of America'], - ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'], - ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'], - ['1.FM Top 40', 'broadcast.content.artist', '\\"Weird Al\\" Yankovic'], - ['Geri Halliwell', 'people.person.profession', 'Actor'], - ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'], - ['My World', 'music.album.artist', 'Justin Bieber'], - ['Don Henley', 'people.person.gender', 'Male'], - ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'], - ['Musician', 'people.profession.specializations', 'Singer'], - ['Die in Your Arms', - 'music.recording.canonical_version', - 'Die in Your Arms'], - ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'], - ['m.0njvs9s', - 'award.award_honor.award', - 'CMT Music Award: Collaborative Video of the Year'], - ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'], - ['Justin Bieber', - 'music.artist.album', - 'Turn to You (Mother’s Day Dedication)'], - ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'], - ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'], - ['City/Town/Village', - 'freebase.type_profile.strict_included_types', - 'Topic'], - ['Recovery', 'common.topic.notable_types', 'Musical Recording'], - ['Dancer', 'common.topic.notable_types', 'Profession'], - ['Live My Life', 'common.topic.notable_types', 'Musical Recording'], - ['Terence Dudley', 'people.person.gender', 'Male'], - ['Baby', 'music.composition.recordings', 'Polka Face'], - ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['BeirutNights.com Radio', - 'broadcast.content.artist', - 'Mr. Sosa & The Yayo'], - ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'], - ['Rihanna', 'music.artist.genre', 'Dance music'], - ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'], - ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'], - ['Gender', 'type.property.expected_type', 'Gender'], - ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'], - ['m.0101fvbf', - 'film.film_regional_release_date.film', - "Justin Bieber's Believe"], - ['m.0yrhrwc', - 'award.award_honor.ceremony', - '2011 MTV Video Music Aid Japan'], - ['MTV Europe Music Award for Best North American Act', - 'award.award_category.winners', - 'm.0yrhmll'], - ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'], - ['m.0101ft1d', - 'film.personal_film_appearance.type_of_appearance', - 'Him/Herself'], - ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'], - ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'], - ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'], - ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'], - ['Ciara', 'people.person.gender', 'Female'], - ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'], - ['Britney Spears', 'music.artist.genre', 'Synthpop'], - ['Thought of You', 'music.recording.artist', 'Justin Bieber'], - ['m.0jzrrqs', - 'location.mailing_address.country', - 'United States of America'], - ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'], - ['Live My Life', 'music.composition.recordings', 'Live My Life'], - ['Toby Gad', 'people.person.nationality', 'United States of America'], - ['Big R Radio - Top 40 Hits', - 'broadcast.content.artist', - 'Natasha Bedingfield'], - ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'], ...], 'choices': []} Although this dataset can be trained on as-is, a couple problems emerge -from doing so: 1. A retrieval algorithm needs to be implemented and +from doing so: +1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond -to the algorithm that was used to generate the dataset subgraphs. 2. The -dataset as is not stored computationally efficiently, as there will +to the algorithm that was used to generate the dataset subgraphs. +2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions. @@ -1526,8 +166,6 @@ along with naively. device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) - print(device) - .. parsed-literal:: @@ -1588,10 +226,6 @@ First, we compare the clock times of encoding using both methods. 121.68579435348511 -.. parsed-literal:: - - - .. code:: ipython3 @@ -1660,229 +294,6 @@ now, the speedup with largegraphindexer will be much higher as the size of the knowledge graph grows. This is due to the speedup being a factor of the number of unique nodes and edges in the graph. -.. code:: ipython3 - - dataset_graphs_embedded_largegraphindexer - - - - -.. parsed-literal:: - - [Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]), - Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]), - Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]), - Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]), - Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]), - Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]), - Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]), - Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]), - Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]), - Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]), - Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]), - Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]), - Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]), - Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]), - Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]), - Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]), - Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]), - Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]), - Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]), - Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]), - Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]), - Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]), - Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]), - Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]), - Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]), - Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]), - Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]), - Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]), - Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]), - Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]), - Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]), - Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]), - Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]), - Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]), - Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]), - Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]), - Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]), - Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]), - Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]), - Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]), - Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]), - Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]), - Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]), - Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]), - Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]), - Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]), - Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]), - Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]), - Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]), - Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]), - Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]), - Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]), - Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]), - Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]), - Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]), - Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]), - Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]), - Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]), - Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]), - Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]), - Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]), - Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]), - Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]), - Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]), - Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]), - Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]), - Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]), - Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]), - Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]), - Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]), - Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]), - Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]), - Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]), - Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]), - Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]), - Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]), - Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]), - Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]), - Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]), - Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]), - Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]), - Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]), - Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]), - Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]), - Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]), - Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]), - Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]), - Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]), - Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]), - Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]), - Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]), - Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]), - Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]), - Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]), - Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]), - Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]), - Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]), - Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]), - Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]), - Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])] - - - -.. code:: ipython3 - - dataset_graphs_embedded - - - - -.. parsed-literal:: - - [Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]), - Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]), - Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]), - Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]), - Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]), - Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]), - Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]), - Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]), - Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]), - Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]), - Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]), - Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]), - Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]), - Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]), - Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]), - Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]), - Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]), - Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]), - Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]), - Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]), - Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]), - Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]), - Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]), - Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]), - Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]), - Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]), - Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]), - Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]), - Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]), - Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]), - Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]), - Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]), - Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]), - Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]), - Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]), - Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]), - Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]), - Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]), - Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]), - Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]), - Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]), - Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]), - Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]), - Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]), - Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]), - Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]), - Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]), - Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]), - Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]), - Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]), - Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]), - Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]), - Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]), - Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]), - Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]), - Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]), - Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]), - Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]), - Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]), - Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]), - Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]), - Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]), - Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]), - Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]), - Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]), - Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]), - Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]), - Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]), - Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]), - Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]), - Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]), - Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]), - Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]), - Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]), - Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]), - Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]), - Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]), - Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]), - Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]), - Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]), - Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]), - Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]), - Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]), - Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]), - Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]), - Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]), - Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]), - Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]), - Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]), - Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]), - Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]), - Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]), - Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]), - Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]), - Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]), - Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]), - Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]), - Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]), - Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]), - Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])] - - We expect the two results to be functionally identical, with the differences being due to floating point jitter. @@ -1955,394 +366,16 @@ the relationships between the entities in the knowledge graph. Building a Multi-Hop QA Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To start, we need to download the raw data of a knowledge graph. In this -case, we use WikiData5M (`Wang et -al `__). -Here we download the raw triplets and their entity codes. Information -about this dataset can be found -`here `__. - -The following download contains the ID to plaintext mapping for all the -entities and relations in the knowledge graph: - -.. code:: ipython3 - - !wget -O "https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz" - -.. code:: ipython3 - - !tar -xvf "wikidata5m_alias.tar.gz" - -.. code:: ipython3 - - with open('wikidata5m_entity.txt') as f: - print(f.readline()) - -.. code:: ipython3 - - with open('wikidata5m_relation.txt') as f: - print(f.readline()) - -And then this download contains the raw triplets: - -.. code:: ipython3 - - !wget -O "https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz" - -.. code:: ipython3 - - !gzip -d "wikidata5m_all_triplet.txt.gz" -f - -.. code:: ipython3 - - with open('wikidata5m_all_triplet.txt') as f: - print(f.readline()) - -To start, we are going to preprocess the knowledge graph to substitute -each of the entity/relation codes with their plaintext aliases. This -makes it easier to use a pre-trained textual encoding model to create -triplet embeddings, as such a model likely won’t understand how to -properly embed the entity codes. - -.. code:: ipython3 - - import pandas as pd - import tqdm - import json - -.. code:: ipython3 - - # Substitute entity codes with their aliases - # Picking the first alias for each entity (rather arbitrarily) - alias_map = {} - rel_alias_map = {} - for line in open('wikidata5m_entity.txt'): - parts = line.strip().split('\t') - entity_id = parts[0] - aliases = parts[1:] - alias_map[entity_id] = aliases[0] - for line in open('wikidata5m_relation.txt'): - parts = line.strip().split('\t') - relation_id = parts[0] - relation_name = parts[1] - rel_alias_map[relation_id] = relation_name - -.. code:: ipython3 - - full_graph = [] - missing_total = 0 - total = 0 - for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')): - src, rel, dst = line.strip().split('\t') - if src not in alias_map: - missing_total += 1 - if dst not in alias_map: - missing_total += 1 - if rel not in rel_alias_map: - missing_total += 1 - total += 3 - full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)]) - print(f"Missing aliases: {missing_total}/{total}") - -.. code:: ipython3 - - full_graph[:10] - -Now ``full_graph`` represents the knowledge graph triplets in -understandable plaintext. - -Next, we need a set of multi-hop questions that the Knowledge Graph will -provide us with context for. We utilize a subset of -`HotPotQA `__ (`Yang et. -al. `__) called -`2WikiMultiHopQA `__ (`Ho et. -al. `__), which -includes a subgraph of entities that serve as the ground truth -justification for answering each multi-hop question: - -.. code:: ipython3 - - !wget -O "https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip" - -.. code:: ipython3 - - !unzip -o "data_ids_april7.zip" - -.. code:: ipython3 - - with open('train.json') as f: - train_data = json.load(f) - train_df = pd.DataFrame(train_data) - train_df['split_type'] = 'train' - - with open('dev.json') as f: - dev_data = json.load(f) - dev_df = pd.DataFrame(dev_data) - dev_df['split_type'] = 'dev' - - with open('test.json') as f: - test_data = json.load(f) - test_df = pd.DataFrame(test_data) - test_df['split_type'] = 'test' - - df = pd.concat([train_df, dev_df, test_df]) - -.. code:: ipython3 - - df.head() - -.. code:: ipython3 - - df['split_type'].value_counts() - -.. code:: ipython3 - - df['type'].value_counts() - -Now we need to extract the subgraphs - -.. code:: ipython3 - - df['graph_size'] = df['evidences_id'].apply(lambda row: len(row)) - -.. code:: ipython3 - - df['graph_size'].value_counts() - -(Optional) We take only questions where the evidence graph is greater -than 0. (Note: this gets rid of the test set): - -.. code:: ipython3 - - # df = df[df['graph_size'] > 0] - -.. code:: ipython3 - - df['split_type'].value_counts() - -.. code:: ipython3 - - df.columns - -.. code:: ipython3 - - refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']] - -.. code:: ipython3 - - refined_df.head() - -Checkpoint: - -.. code:: ipython3 - - refined_df.to_csv('wikimultihopqa_refined.csv', index=False) - -Now we need to check that all the entities mentioned in the -question/answer set are also present in the Wikidata graph: - -.. code:: ipython3 - - relation_map = {} - with open('wikidata5m_relation.txt') as f: - for line in tqdm.tqdm(f): - parts = line.strip().split('\t') - for i in range(1, len(parts)): - if parts[i] not in relation_map: - relation_map[parts[i]] = [] - relation_map[parts[i]].append(parts[0]) - -.. code:: ipython3 - - # Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M - for row in refined_df.itertuples(): - for trip in row.evidences_id: - relation = trip[1] - if relation not in relation_map: - print(f'The following relation is not found: {relation}') - elif len(relation_map[relation]) > 1: - print(f'The following relation alias has a collision: {relation}: {relation_map[relation]}') - -.. code:: ipython3 - - entity_set = set() - with open('wikidata5m_entity.txt') as f: - for line in tqdm.tqdm(f): - entity_set.add(line.strip().split('\t')[0]) - -.. code:: ipython3 - - missing_entities = set() - missing_entity_idx = set() - for i, row in enumerate(refined_df.itertuples()): - for trip in row.evidences_id: - if len(trip) != 3: - print(trip) - entities = trip[0], trip[2] - for entity in entities: - if entity not in entity_set: - print(f'The following entity was not found in the KG: {entity}') - missing_entities.add(entity) - missing_entity_idx.add(i) - -Right now, we drop the missing entity entries. Additional preprocessing -can be done here to resolve the entity/relation collisions, but that is -out of the scope for this notebook. - -.. code:: ipython3 - - len(missing_entity_idx) - -.. code:: ipython3 - - refined_df.shape - -.. code:: ipython3 - - # missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped. - refined_df.reset_index(inplace=True, drop=True) - refined_df - -.. code:: ipython3 - - cleaned_df = refined_df.drop(missing_entity_idx) - cleaned_df['split_type'].value_counts() - -Now we save the resulting graph and questions/answers dataset: - -.. code:: ipython3 - - cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False) - -.. code:: ipython3 - - import torch - -.. code:: ipython3 - - torch.save(full_graph, 'wikimultihopqa_full_graph.pt') +To see an example of encoding a large knowledge graph starting from an +existing set of triplets, check out the multi-hop example in +`examples/llm_plus_gnn/multihop`. Question: How do we extract a contextual subgraph for a given query? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The chosen retrieval algorithm is a critical component in the pipeline -for affecting RAG performance. In the next section (1), we will -demonstrate a naive method of retrieval for a large knowledge graph, and -how to apply it to this dataset along with WebQSP. - -Preparing a Textualized Graph for LLM -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For now however, we need to prepare the graph data to be used as a -plaintext prefix to the LLM. In order to do this, we want to prompt the -LLM to use the unique nodes, and unique edge triplets of a given -subgraph. In order to do this, we prepare a unique indexed node df and -edge df for the knowledge graph now. This process occurs trivially with -the LargeGraphIndexer: - -.. code:: ipython3 - - from torch_geometric.data import LargeGraphIndexer - -.. code:: ipython3 - - indexer = LargeGraphIndexer.from_triplets(full_graph) - -.. code:: ipython3 - - # Node DF - textual_nodes = pd.DataFrame.from_dict( - {"node_attr": indexer.get_node_features()}) - textual_nodes["node_id"] = textual_nodes.index - textual_nodes = textual_nodes[["node_id", "node_attr"]] - -.. code:: ipython3 - - textual_nodes.head() - -Notice how LargeGraphIndexer ensures that there are no duplicate -indices: - -.. code:: ipython3 - - textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0] - -.. code:: ipython3 - - # Edge DF - textual_edges = pd.DataFrame(indexer.get_edge_features(), - columns=["src", "edge_attr", "dst"]) - textual_edges["src"] = [ - indexer._nodes[h] for h in textual_edges["src"] - ] - textual_edges["dst"] = [ - indexer._nodes[h] for h in textual_edges["dst"] - ] - -Note: The edge table refers to each node by its index in the node table. -We will see how this gets utilized later when indexing a subgraph. - -.. code:: ipython3 - - textual_edges.head() - -Now we can save the result - -.. code:: ipython3 - - textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False) - textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False) - -Now were done! This knowledge graph and dataset will get used later on -in Section 1. - -.. code:: ipython3 - - # TODO: Refactor everything below this point into its own notebook - -Generating Subgraphs -==================== - -.. code:: ipython3 - - from profiling_utils import create_remote_backend_from_triplets - from rag_feature_store import SentenceTransformerFeatureStore - from rag_graph_store import NeighborSamplingRAGGraphStore - from torch_geometric.loader import RAGQueryLoader - from torch_geometric.nn.nlp import SentenceTransformer - from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst - from torch_geometric.data import get_features_for_triplets_groups, Data - -.. code:: ipython3 - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) - -.. code:: ipython3 - - fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={"batch_size": 256}).load() - -.. code:: ipython3 - - import torch - import pandas as pd - -.. code:: ipython3 - - graphs = torch.load('subg_results.pt') - graph_df = pd.read_csv("wikimultihopqa_cleaned.csv") - -.. code:: ipython3 - - graph_df['is_train'].value_counts() - -.. code:: ipython3 - - graph_df.head() - -.. code:: ipython3 - - graph_df['is_train'].value_counts() +for affecting RAG performance. In the next section, we will +demonstrate a naive method of retrieval for a large knowledge graph. Retrieval Algorithms and Scaling Retrieval @@ -2352,13 +385,13 @@ Motivation ---------- When building a RAG Pipeline for inference, the retrieval component is -important for the following reasons: 1. A given algorithm for retrieving -subgraph context can have a marked effect on the hallucination rate of -the responses in the model 2. A given retrieval algorithm needs to be -able to scale to larger graphs of millions of nodes and edges in order -to be practical for production. +important for the following reasons: +1. A given algorithm for retrieving subgraph context can have a +marked effect on the hallucination rate of the responses in the model +2. A given retrieval algorithm needs to be able to scale to larger +graphs of millions of nodes and edges in order to be practical for production. -In this notebook, we will explore how to construct a RAG retrieval +In this section, we will explore how to construct a RAG retrieval algorithm from a given subgraph, and conduct some experiments to evaluate its runtime performance. @@ -2391,15 +424,8 @@ experiment: from itertools import chain -.. parsed-literal:: - - /home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html - from .autonotebook import tqdm as notebook_tqdm - - .. code:: ipython3 - # Limiting to 10 questions for the sake of compute, but can be increased if necessary ds = WebQSPDataset(root='demo', limit=10) Let’s set up our set of questions and graph triplets: @@ -2455,11 +481,10 @@ Let’s set up our set of questions and graph triplets: all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset)) -With these questions and triplets, we want to: 1. Consolidate all the -relations in these triplets into a Knowledge Graph 2. Create a -FeatureStore that encodes all the nodes and edges in the knowledge graph -3. Create a GraphStore that encodes all the edge indices in the -knowledge graph +With these questions and triplets, we want to: +1. Consolidate all the relations in these triplets into a Knowledge Graph +2. Create a FeatureStore that encodes all the nodes and edges in the knowledge graph +3. Create a GraphStore that encodes all the edge indices in the knowledge graph .. code:: ipython3 @@ -2556,10 +581,10 @@ look at the retrieval process for a remote backend: As we see here, there are 3 important steps to any remote backend -procedure for graphs: 1. Retrieve the seed nodes and edges to begin our -retrieval process from. 2. Traverse the graph neighborhood of the seed -nodes/edges to gather local context. 3. Fetch the features associated -with the subgraphs obtained from the traversal. +procedure for graphs: +1. Retrieve the seed nodes and edges to begin our retrieval process from. +2. Traverse the graph neighborhood of the seed nodes/edges to gather local context. +3. Fetch the features associated with the subgraphs obtained from the traversal. We can see that our Query Loader construction allows us to specify unique hyperparameters for each unique step in this retrieval. @@ -2578,15 +603,6 @@ subgraphs: sub_graphs.append(query_loader.query(q)) -.. parsed-literal:: - - 0%| | 0/10 [00:00 Date: Thu, 29 Aug 2024 06:02:12 -0700 Subject: [PATCH 701/752] add images --- docs/source/_figures/flowchart.svg | 1 + docs/source/_figures/multihop_example.svg | 1 + docs/source/_figures/remote_backend.svg | 1 + docs/source/advanced/rag.rst | 60 ++++++----------------- docs/source/index.rst | 1 + 5 files changed, 20 insertions(+), 44 deletions(-) create mode 100644 docs/source/_figures/flowchart.svg create mode 100644 docs/source/_figures/multihop_example.svg create mode 100644 docs/source/_figures/remote_backend.svg diff --git a/docs/source/_figures/flowchart.svg b/docs/source/_figures/flowchart.svg new file mode 100644 index 000000000000..188d37b14f41 --- /dev/null +++ b/docs/source/_figures/flowchart.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_figures/multihop_example.svg b/docs/source/_figures/multihop_example.svg new file mode 100644 index 000000000000..42e2eaf36afa --- /dev/null +++ b/docs/source/_figures/multihop_example.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_figures/remote_backend.svg b/docs/source/_figures/remote_backend.svg new file mode 100644 index 000000000000..c5791f0a95de --- /dev/null +++ b/docs/source/_figures/remote_backend.svg @@ -0,0 +1 @@ + diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 06ce71b7d42d..9c172af13801 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -32,7 +32,9 @@ nodes, we present the following architecture: -.. image:: _Introduction_files/_Introduction_7_0.svg +.. figure:: ../_figures/flowchart.svg + :align: center + :width: 100% @@ -279,11 +281,6 @@ First, we compare the clock times of encoding using both methods. 114.66037964820862 -.. parsed-literal:: - - - - The large graph indexer allows us to compute the entire knowledge graph from a series of samples, so that new retrieval methods can also be tested on the entire graph. We will see this attempted in practice later @@ -345,18 +342,10 @@ hops apart. This can be challenging for an LLM to recognize from prepended graph information. Here’s a motivating example (credit to @Rishi Puri): -.. code:: ipython3 - - from IPython.display import SVG - -.. code:: ipython3 - - SVG(filename='./media/multihop_example.svg') - - - -.. image:: 0_1_Encoding_from_Scratch_files/0_1_Encoding_from_Scratch_6_0.svg +.. figure:: ../_figures/multihop_example.svg + :align: center + :width: 100% @@ -398,15 +387,10 @@ evaluate its runtime performance. We want to do so in-line with Pytorch Geometric’s in-house framework for remote backends: -.. code:: ipython3 - - from IPython.display import Image, SVG - Image(filename='../../../docs/source/_figures/remote_2.png') - - - -.. image:: 1_Retrieval_files/1_Retrieval_5_0.png +.. figure:: ../_figures/remote_2.png + :align: center + :width: 100% @@ -490,7 +474,7 @@ With these questions and triplets, we want to: import torch from torch_geometric.nn.nlp import SentenceTransformer - from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet + from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet .. code:: ipython3 @@ -535,14 +519,10 @@ Now that we have initialized our remote backends, we can now retrieve from them using a Loader to query the backends, as shown in this diagram: -.. code:: ipython3 - - Image(filename='../../../docs/source/_figures/remote_3.png') - - - -.. image:: 1_Retrieval_files/1_Retrieval_21_0.png +.. figure:: ../_figures/remote_3.png + :align: center + :width: 100% @@ -569,14 +549,10 @@ diagram: To make better sense of this loader’s arguments, let’s take a closer look at the retrieval process for a remote backend: -.. code:: ipython3 - - SVG(filename="media/remote_backend.svg") - - - -.. image:: 1_Retrieval_files/1_Retrieval_25_0.svg +.. figure:: ../_figures/remote_backend.svg + :align: center + :width: 100% @@ -761,10 +737,6 @@ method? Let’s compare some basics properties of the subgraphs: Size: 1817, Ground Truth Size: 1978, Accuracy: 0.7530475815965395, Precision: 0.1677807486631016, Recall: 0.11696178937558248 -.. parsed-literal:: - - - Note that, since we’re only comparing the results of 10 graphs here, this retrieval algorithm is not taking into account the full corpus of diff --git a/docs/source/index.rst b/docs/source/index.rst index 1c67eeddeec6..02315461c6ec 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,6 +44,7 @@ In addition, it consists of easy-to-use mini-batch loaders for operating on many advanced/remote advanced/graphgym advanced/cpu_affinity + advanced/rag .. toctree:: :maxdepth: 1 From 2a490226ed49a9aad0ba8d8f78335c17992baf6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:05:20 +0000 Subject: [PATCH 702/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/advanced/rag.rst | 60 +++++++------- .../llm_plus_gnn/benchmark_model_archs.py | 26 ++++--- .../multihop/multihop_preprocess.py | 28 ++++--- .../multihop/rag_generate_multihop.py | 20 +++-- .../nvtx_examples/nvtx_rag_backend_example.py | 78 +++++++++++++------ .../nvtx_examples/nvtx_webqsp_example.py | 15 ++-- examples/llm_plus_gnn/rag_feature_store.py | 44 +++++++---- examples/llm_plus_gnn/rag_generate.py | 75 ++++++++++++------ test/datasets/test_web_qsp_dataset.py | 5 +- test/nn/models/test_g_retriever.py | 10 ++- torch_geometric/data/large_graph_indexer.py | 6 +- torch_geometric/loader/rag_loader.py | 22 ++++-- torch_geometric/nn/models/g_retriever.py | 11 ++- 13 files changed, 256 insertions(+), 144 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 9c172af13801..595db8d22632 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -1,7 +1,7 @@ Working with LLM RAG in Pytorch Geometric ========================================= -This series aims to provide a starting point and for +This series aims to provide a starting point and for multi-step LLM Retrieval Augmented Generation (RAG) using Graph Neural Networks. @@ -60,7 +60,7 @@ are composed of the following steps: Encoding a Large Knowledge Graph ================================ -To start, a Large Knowledge Graph needs to be created from triplets or +To start, a Large Knowledge Graph needs to be created from triplets or multiple subgraphs in a dataset. Example 1: Building from Already Existing Datasets @@ -68,13 +68,13 @@ Example 1: Building from Already Existing Datasets In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. -The same is true for GNN based RAG. For example, consider the +The same is true for GNN based RAG. For example, consider the WebQSPDataset. .. code:: ipython3 from torch_geometric.datasets import WebQSPDataset - + num_questions = 100 ds = WebQSPDataset('small_sample', limit=num_questions) @@ -129,7 +129,7 @@ Although this dataset can be trained on as-is, a couple problems emerge from doing so: 1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond -to the algorithm that was used to generate the dataset subgraphs. +to the algorithm that was used to generate the dataset subgraphs. 2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions. @@ -185,37 +185,37 @@ First, we compare the clock times of encoding using both methods. nodes_map = dict() edges_map = dict() edge_idx_base = [] - + for src, edge, dst in graph: # Collect nodes if src not in nodes_map: nodes_map[src] = len(nodes_map) if dst not in nodes_map: nodes_map[dst] = len(nodes_map) - + # Collect edge types if edge not in edges_map: edges_map[edge] = len(edges_map) - + # Record edge edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst])) - + # Encode nodes and edges sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x])) sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x])) - + x = model.encode(sorted_nodes, batch_size=256) edge_attrs_map = model.encode(sorted_edges, batch_size=256) - + edge_attrs = [] edge_idx = [] for trip in edge_idx_base: edge_attrs.append(edge_attrs_map[trip[1]]) edge_idx.append([trip[0], trip[2]]) - + dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0))) - - + + print(time.time()-start) @@ -233,31 +233,31 @@ First, we compare the clock times of encoding using both methods. # Using LargeGraphIndexer to make one large knowledge graph from torch_geometric.data.large_graph_indexer import EDGE_RELATION - + start = time.time() all_triplets_together = chain.from_iterable(raw_dataset_graphs) # Index as one large graph print('Indexing...') indexer = LargeGraphIndexer.from_triplets(all_triplets_together) - + # first the nodes unique_nodes = indexer.get_unique_node_features() node_encs = model.encode(unique_nodes, batch_size=256) indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs) - + # then the edges unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION) edge_attr = model.encode(unique_edges, batch_size=256) indexer.add_edge_feature(new_feature_name="edge_attr", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION) - + ckpt_time = time.time() - whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') + whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') whole_graph_done = time.time() print(f"Time to create whole knowledge_graph: {whole_graph_done-start}") - + # Compute this to make sure we're comparing like to like on final time printout whole_graph_diff = whole_graph_done-ckpt_time - + # retrieve subgraphs print('Retrieving Subgraphs...') dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)] @@ -356,7 +356,7 @@ Building a Multi-Hop QA Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To see an example of encoding a large knowledge graph starting from an -existing set of triplets, check out the multi-hop example in +existing set of triplets, check out the multi-hop example in `examples/llm_plus_gnn/multihop`. Question: How do we extract a contextual subgraph for a given query? @@ -374,8 +374,8 @@ Motivation ---------- When building a RAG Pipeline for inference, the retrieval component is -important for the following reasons: -1. A given algorithm for retrieving subgraph context can have a +important for the following reasons: +1. A given algorithm for retrieving subgraph context can have a marked effect on the hallucination rate of the responses in the model 2. A given retrieval algorithm needs to be able to scale to larger graphs of millions of nodes and edges in order to be practical for production. @@ -488,11 +488,11 @@ from triplets: .. code:: ipython3 from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader - + # We define this GraphStore to sample the neighbors of a node locally. # Ideally for a real remote backend, this interface would be replaced with an API to a Graph DB, such as Neo4j. from rag_graph_store import NeighborSamplingRAGGraphStore - + # We define this FeatureStore to encode the nodes and edges locally, and perform appoximate KNN when indexing. # Ideally for a real remote backend, this interface would be replaced with an API to a vector DB, such as Pinecone. from rag_feature_store import SentenceTransformerFeatureStore @@ -501,7 +501,7 @@ from triplets: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = SentenceTransformer(model_name="sentence-transformers/all-roberta-large-v1").to(device) - + backend_loader: RemoteGraphBackendLoader = create_remote_backend_from_triplets( triplets=all_triplets, # All the triplets to insert into the backend node_embedding_model=model, # Embedding model to process triplets with @@ -511,7 +511,7 @@ from triplets: node_method_kwargs={"batch_size": 256}, # Keyword arguments to pass to the node_method_to_call. graph_db=NeighborSamplingRAGGraphStore, # Graph Store to use feature_db=SentenceTransformerFeatureStore # Feature Store to use - ) + ) # This loader saves a copy of the processed data locally to be transformed into a graphstore and featurestore when load() is called. feature_store, graph_store = backend_loader.load() @@ -536,7 +536,7 @@ diagram: data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store # Arguments to pass into the seed node/edge retrieval methods for the FeatureStore. # In this case, it's k for the KNN on the nodes and edges. - seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, + seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, # Arguments to pass into the GraphStore's Neighbor sampling method. # In this case, the GraphStore implements a NeighborLoader, so it takes the same arguments. sampler_kwargs={"num_neighbors": [40]*3}, @@ -557,7 +557,7 @@ look at the retrieval process for a remote backend: As we see here, there are 3 important steps to any remote backend -procedure for graphs: +procedure for graphs: 1. Retrieve the seed nodes and edges to begin our retrieval process from. 2. Traverse the graph neighborhood of the seed nodes/edges to gather local context. 3. Fetch the features associated with the subgraphs obtained from the traversal. diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index e3519fd00406..1a4184344cab 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -1,14 +1,15 @@ -"""Used to benchmark the performance of an untuned/fine tuned LLM against -GRetriever with various architectures and layer depths.""" +"""Used to benchmark the performance of an untuned/fine tuned LLM against +GRetriever with various architectures and layer depths. +""" # %% import argparse import torch +from g_retriever import benchmark_models, get_loss, inference_step + from torch_geometric.datasets import UpdatedWebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever -from g_retriever import benchmark_models, get_loss, inference_step - # %% parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") parser.add_argument("--hidden_channels", type=int, default=1024) @@ -45,17 +46,22 @@ class MockDataset: """Utility class to patch the fields in WebQSPDataset used by GRetriever.""" def __init__(self) -> None: pass - + @property def split_idxs(self) -> dict: # Imitates the WebQSP split method return { - "train": torch.arange(args.num_train), - "val": torch.arange(args.num_val) + args.num_train, - "test": torch.arange(args.num_test) + args.num_train + args.num_val, + "train": + torch.arange(args.num_train), + "val": + torch.arange(args.num_val) + args.num_train, + "test": + torch.arange(args.num_test) + args.num_train + args.num_val, } + def __getitem__(self, idx: int): return dataset[idx] + ds = MockDataset() # %% @@ -74,7 +80,8 @@ def __getitem__(self, idx: int): model_names.append(f"{m_type}_{n_layer}_{n_tokens}") model_classes.append(GRetriever) kwargs = dict(gnn_hidden_channels=hidden_channels, - num_gnn_layers=n_layer, gnn_to_use=models[m_type], mlp_out_tokens=n_tokens) + num_gnn_layers=n_layer, gnn_to_use=models[m_type], + mlp_out_tokens=n_tokens) if args.tiny_llama: kwargs['llm_to_use'] = 'TinyLlama/TinyLlama-1.1B-Chat-v0.1' kwargs['mlp_out_dim'] = 2048 @@ -85,4 +92,3 @@ def __getitem__(self, idx: int): benchmark_models(model_classes, model_names, model_kwargs, ds, lr, epochs, batch_size, eval_batch_size, get_loss, inference_step, skip_LLMs=False, tiny_llama=args.tiny_llama, force=True) - diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop/multihop_preprocess.py index 4272361d59da..9d85c85f5034 100644 --- a/examples/llm_plus_gnn/multihop/multihop_preprocess.py +++ b/examples/llm_plus_gnn/multihop/multihop_preprocess.py @@ -28,16 +28,18 @@ # The following download contains the ID to plaintext mapping for all the entities and relations in the knowledge graph: from subprocess import call + rv = call("./multihop_download.sh") # %% [markdown] # To start, we are going to preprocess the knowledge graph to substitute each of the entity/relation codes with their plaintext aliases. This makes it easier to use a pre-trained textual encoding model to create triplet embeddings, as such a model likely won't understand how to properly embed the entity codes. +import argparse +import json + # %% import pandas as pd import tqdm -import json -import argparse # %% parser = argparse.ArgumentParser(description="Preprocess wikidata5m") @@ -78,7 +80,11 @@ if rel not in rel_alias_map: missing_total += 1 total += 3 - full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)]) + full_graph.append([ + alias_map.get(src, src), + rel_alias_map.get(rel, rel), + alias_map.get(dst, dst) + ]) i += 1 print(f"Missing aliases: {missing_total}/{total}") @@ -119,7 +125,10 @@ # df = df[df['graph_size'] > 0] # %% -refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']] +refined_df = df[[ + '_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', + 'graph_size' +]] # %% [markdown] # Checkpoint: @@ -205,20 +214,15 @@ textual_nodes["node_id"] = textual_nodes.index textual_nodes = textual_nodes[["node_id", "node_attr"]] - # %% [markdown] # Notice how LargeGraphIndexer ensures that there are no duplicate indices: # %% # Edge DF textual_edges = pd.DataFrame(indexer.get_edge_features(), - columns=["src", "edge_attr", "dst"]) -textual_edges["src"] = [ - indexer._nodes[h] for h in textual_edges["src"] -] -textual_edges["dst"] = [ - indexer._nodes[h] for h in textual_edges["dst"] -] + columns=["src", "edge_attr", "dst"]) +textual_edges["src"] = [indexer._nodes[h] for h in textual_edges["src"]] +textual_edges["dst"] = [indexer._nodes[h] for h in textual_edges["dst"]] # %% [markdown] # Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph. diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index 5c2b37fa7dd1..a0419deff36d 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -1,9 +1,13 @@ # %% +import sys + import pandas as pd import torch import tqdm -import sys + sys.path.append('..') +import argparse + from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerApproxFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore @@ -14,10 +18,10 @@ ) from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer -import argparse # %% -parser = argparse.ArgumentParser(description="Generate new multihop dataset for rag") +parser = argparse.ArgumentParser( + description="Generate new multihop dataset for rag") # TODO: Add more arguments for configuring rag params parser.add_argument("--num_samples", type=int) args = parser.parse_args() @@ -45,19 +49,25 @@ # %% from typing import Tuple + from torch_geometric.data import Data all_textual_nodes = pd.read_csv('wikimultihopqa_textual_nodes.csv') all_textual_edges = pd.read_csv('wikimultihopqa_textual_edges.csv') -def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: q_emb = model.encode(query) textual_nodes = all_textual_nodes.iloc[graph["node_idx"]].reset_index() textual_edges = all_textual_edges.iloc[graph["edge_idx"]].reset_index() - out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, + textual_edges, topk, topk_e, cost_e) out_graph["desc"] = desc return out_graph + # %% query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index 77ff8d214c28..69b1602e4316 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -1,38 +1,46 @@ # %% +import argparse +from itertools import chain +from typing import Tuple + +import torch from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import rag_loader + +from torch_geometric.data import Data, get_features_for_triplets_groups from torch_geometric.datasets import WebQSPDataset -from torch_geometric.nn.nlp import SentenceTransformer from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain +from torch_geometric.loader import rag_loader +from torch_geometric.nn.nlp import SentenceTransformer from torch_geometric.profile.nvtx import nvtxit -import torch -import argparse -from typing import Tuple # %% # Patch FeatureStore and GraphStore -SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_nodes) -SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()(SentenceTransformerFeatureStore.retrieve_seed_edges) -SentenceTransformerFeatureStore.load_subgraph = nvtxit()(SentenceTransformerFeatureStore.load_subgraph) -NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()(NeighborSamplingRAGGraphStore.sample_subgraph) +SentenceTransformerFeatureStore.retrieve_seed_nodes = nvtxit()( + SentenceTransformerFeatureStore.retrieve_seed_nodes) +SentenceTransformerFeatureStore.retrieve_seed_edges = nvtxit()( + SentenceTransformerFeatureStore.retrieve_seed_edges) +SentenceTransformerFeatureStore.load_subgraph = nvtxit()( + SentenceTransformerFeatureStore.load_subgraph) +NeighborSamplingRAGGraphStore.sample_subgraph = nvtxit()( + NeighborSamplingRAGGraphStore.sample_subgraph) rag_loader.RAGQueryLoader.query = nvtxit()(rag_loader.RAGQueryLoader.query) # %% ds = WebQSPDataset("small_ds_1", force_reload=True, limit=10) # %% -triplets = list(chain.from_iterable((d['graph'] for d in ds.raw_dataset))) +triplets = list(chain.from_iterable(d['graph'] for d in ds.raw_dataset)) # %% questions = ds.raw_dataset['question'] # %% -ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) +ground_truth_graphs = get_features_for_triplets_groups( + ds.indexer, (d['graph'] for d in ds.raw_dataset), + pre_transform=preprocess_triplet) num_edges = len(ds.indexer._edges) # %% @@ -40,22 +48,38 @@ model = SentenceTransformer().to(device) # %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() +fs, gs = create_remote_backend_from_triplets( + triplets=triplets, node_embedding_model=model, + node_method_to_call="encode", path="backend", + pre_transform=preprocess_triplet, node_method_kwargs={ + "batch_size": 256 + }, graph_db=NeighborSamplingRAGGraphStore, + feature_db=SentenceTransformerFeatureStore).load() # %% from torch_geometric.datasets.web_qsp_dataset import retrieval_via_pcst + @nvtxit() -def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: q_emb = model.encode(query) textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() - out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, + textual_edges, topk, topk_e, cost_e) out_graph["desc"] = desc return graph + # %% -query_loader = rag_loader.RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 10}, seed_edges_kwargs={"k_edges": 10}, sampler_kwargs={"num_neighbors": [40]*10}, local_filter=apply_retrieval_via_pcst) +query_loader = rag_loader.RAGQueryLoader( + data=(fs, gs), seed_nodes_kwargs={"k_nodes": + 10}, seed_edges_kwargs={"k_edges": 10}, + sampler_kwargs={"num_neighbors": + [40] * 10}, local_filter=apply_retrieval_via_pcst) + # %% # Accuracy Metrics to be added to Profiler @@ -68,15 +92,21 @@ def _eidx_helper(subg: Data, ground_truth: Data): subg_e = set(subg_eidx) gt_e = set(gt_eidx) return subg_e, gt_e + + def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): subg_e, gt_e = _eidx_helper(subg, ground_truth) total_e = set(range(num_edges)) tp = len(subg_e & gt_e) - tn = len(total_e-(subg_e | gt_e)) - return (tp+tn)/num_edges + tn = len(total_e - (subg_e | gt_e)) + return (tp + tn) / num_edges + + def check_retrieval_precision(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(subg_e) + + def check_retrieval_recall(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) @@ -84,10 +114,14 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): # %% + @nvtxit() def _run_eval(): - for subg, gt in zip((query_loader.query(q) for q in questions), ground_truth_graphs): - print(check_retrieval_accuracy(subg, gt, num_edges), check_retrieval_precision(subg, gt), check_retrieval_recall(subg, gt)) + for subg, gt in zip((query_loader.query(q) for q in questions), + ground_truth_graphs): + print(check_retrieval_accuracy(subg, gt, num_edges), + check_retrieval_precision(subg, gt), + check_retrieval_recall(subg, gt)) if __name__ == "__main__": @@ -98,4 +132,4 @@ def _run_eval(): with torch.autograd.profiler.emit_nvtx(): _run_eval() else: - _run_eval() \ No newline at end of file + _run_eval() diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py index 1db8e5934728..145cc00a410b 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py @@ -1,12 +1,15 @@ +import argparse + +import torch + from torch_geometric.datasets import web_qsp_dataset from torch_geometric.profile import nvtxit -import torch -import argparse # Apply Patches -web_qsp_dataset.retrieval_via_pcst = nvtxit()(web_qsp_dataset.retrieval_via_pcst) -web_qsp_dataset.WebQSPDataset.process = nvtxit()(web_qsp_dataset.WebQSPDataset.process) - +web_qsp_dataset.retrieval_via_pcst = nvtxit()( + web_qsp_dataset.retrieval_via_pcst) +web_qsp_dataset.WebQSPDataset.process = nvtxit()( + web_qsp_dataset.WebQSPDataset.process) if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -16,4 +19,4 @@ with torch.autograd.profiler.emit_nvtx(): ds = web_qsp_dataset.WebQSPDataset('baseline') else: - ds = web_qsp_dataset.WebQSPDataset('baseline') \ No newline at end of file + ds = web_qsp_dataset.WebQSPDataset('baseline') diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index 7e2e3fa541b4..b9b60a0070c6 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -1,3 +1,4 @@ +import gc from collections.abc import Iterable, Iterator from typing import Any, Dict, Optional, Type, Union @@ -9,11 +10,9 @@ from torch_geometric.data import Data, HeteroData from torch_geometric.distributed import LocalFeatureStore from torch_geometric.nn.nlp import SentenceTransformer +from torch_geometric.nn.pool import ApproxMIPSKNNIndex from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput from torch_geometric.typing import InputEdges, InputNodes -import gc -from torch_geometric.nn.pool import ApproxMIPSKNNIndex - # NOTE: Only compatible with Homogeneous graphs for now @@ -57,7 +56,7 @@ def _retrieve_seed_nodes_batch(self, query: Iterable[Any], yield indices def retrieve_seed_edges(self, query: Any, k_edges: int = 3) -> InputEdges: - result = next(self._retrieve_seed_edges_batch([query], k_edges)) + result = next(self._retrieve_seed_edges_batch([query], k_edges)) gc.collect() torch.cuda.empty_cache() return result @@ -98,7 +97,9 @@ def load_subgraph( # TODO: Refactor because composition >> inheritance -def _add_features_to_knn_index(knn_index: ApproxMIPSKNNIndex, emb: Tensor, device: torch.device, batch_size: int = 2**20): + +def _add_features_to_knn_index(knn_index: ApproxMIPSKNNIndex, emb: Tensor, + device: torch.device, batch_size: int = 2**20): """Add new features to the existing KNN index in batches. Args: @@ -109,11 +110,12 @@ def _add_features_to_knn_index(knn_index: ApproxMIPSKNNIndex, emb: Tensor, devic """ for i in range(0, emb.size(0), batch_size): if emb.size(0) - i >= batch_size: - emb_batch = emb[i:i+batch_size].to(device) + emb_batch = emb[i:i + batch_size].to(device) else: - emb_batch = emb[i:].to(device) + emb_batch = emb[i:].to(device) knn_index.add(emb_batch) + class ApproxKNNRAGFeatureStore(KNNRAGFeatureStore): def __init__(self, enc_model: Type[Module], model_kwargs: Optional[Dict[str, @@ -130,15 +132,18 @@ def _retrieve_seed_nodes_batch(self, query: Iterable[Any], enc_model = self.enc_model.to(self.device) query_enc = enc_model.encode(query, - **self.model_kwargs).to(self.device) + **self.model_kwargs).to(self.device) del enc_model gc.collect() torch.cuda.empty_cache() if self.node_knn_index is None: - self.node_knn_index = ApproxMIPSKNNIndex(num_cells=100, num_cells_to_visit=100, bits_per_vector=4) + self.node_knn_index = ApproxMIPSKNNIndex(num_cells=100, + num_cells_to_visit=100, + bits_per_vector=4) # Need to add in batches to avoid OOM - _add_features_to_knn_index(self.node_knn_index, self.x, self.device) + _add_features_to_knn_index(self.node_knn_index, self.x, + self.device) output = self.node_knn_index.search(query_enc, k=k_nodes) yield from output.index @@ -150,28 +155,33 @@ def _retrieve_seed_edges_batch(self, query: Iterable[Any], enc_model = self.enc_model.to(self.device) query_enc = enc_model.encode(query, - **self.model_kwargs).to(self.device) + **self.model_kwargs).to(self.device) del enc_model gc.collect() torch.cuda.empty_cache() if self.edge_knn_index is None: - self.edge_knn_index = ApproxMIPSKNNIndex(num_cells=100, num_cells_to_visit=100, bits_per_vector=4) + self.edge_knn_index = ApproxMIPSKNNIndex(num_cells=100, + num_cells_to_visit=100, + bits_per_vector=4) # Need to add in batches to avoid OOM - _add_features_to_knn_index(self.edge_knn_index, self.edge_attr, self.device) + _add_features_to_knn_index(self.edge_knn_index, self.edge_attr, + self.device) output = self.edge_knn_index.search(query_enc, k=k_edges) yield from output.index - # TODO: These two classes should be refactored class SentenceTransformerFeatureStore(KNNRAGFeatureStore): def __init__(self, *args, **kwargs): - kwargs['model_name'] = kwargs.get('model_name', 'sentence-transformers/all-roberta-large-v1') + kwargs['model_name'] = kwargs.get( + 'model_name', 'sentence-transformers/all-roberta-large-v1') super().__init__(SentenceTransformer, *args, **kwargs) + class SentenceTransformerApproxFeatureStore(ApproxKNNRAGFeatureStore): def __init__(self, *args, **kwargs): - kwargs['model_name'] = kwargs.get('model_name','sentence-transformers/all-roberta-large-v1') - super().__init__(SentenceTransformer, *args, **kwargs) \ No newline at end of file + kwargs['model_name'] = kwargs.get( + 'model_name', 'sentence-transformers/all-roberta-large-v1') + super().__init__(SentenceTransformer, *args, **kwargs) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index f3303a97a5b8..e75eb3b311b2 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -1,18 +1,23 @@ # %% +import argparse +from itertools import chain +from typing import Tuple + +import pandas as pd +import torch +import tqdm from profiling_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore -from torch_geometric.loader import RAGQueryLoader + +from torch_geometric.data import Data from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.datasets.web_qsp_dataset import ( + preprocess_triplet, + retrieval_via_pcst, +) +from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer -from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet, retrieval_via_pcst -from torch_geometric.data import get_features_for_triplets_groups, Data -from itertools import chain -import torch -from typing import Tuple -import tqdm -import pandas as pd -import argparse # %% parser = argparse.ArgumentParser(description="Generate new WebQSP subgraphs") @@ -26,39 +31,57 @@ ds = UpdatedWebQSPDataset("dataset", limit=args.num_samples) # %% -triplets = chain.from_iterable((d['graph'] for d in ds.raw_dataset)) +triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) # %% questions = ds.raw_dataset['question'] # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) +model = SentenceTransformer( + model_name='sentence-transformers/all-roberta-large-v1').to(device) # %% -fs, gs = create_remote_backend_from_triplets(triplets=triplets, node_embedding_model=model, node_method_to_call="encode", path="backend", pre_transform=preprocess_triplet, node_method_kwargs={"batch_size": 256}, graph_db=NeighborSamplingRAGGraphStore, feature_db=SentenceTransformerFeatureStore).load() +fs, gs = create_remote_backend_from_triplets( + triplets=triplets, node_embedding_model=model, + node_method_to_call="encode", path="backend", + pre_transform=preprocess_triplet, node_method_kwargs={ + "batch_size": 256 + }, graph_db=NeighborSamplingRAGGraphStore, + feature_db=SentenceTransformerFeatureStore).load() # %% -def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, topk_e: int = 3, cost_e: float = 0.5) -> Tuple[Data, str]: + +def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, + topk_e: int = 3, + cost_e: float = 0.5) -> Tuple[Data, str]: q_emb = model.encode(query) textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() - out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, textual_edges, topk, topk_e, cost_e) + out_graph, desc = retrieval_via_pcst(graph, q_emb, textual_nodes, + textual_edges, topk, topk_e, cost_e) out_graph["desc"] = desc return out_graph + def apply_retrieval_with_text(graph: Data, query: str) -> Tuple[Data, str]: textual_nodes = ds.textual_nodes.iloc[graph["node_idx"]].reset_index() textual_edges = ds.textual_edges.iloc[graph["edge_idx"]].reset_index() - desc = (textual_nodes.to_csv(index=False) + "\n" + - textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"])) + desc = ( + textual_nodes.to_csv(index=False) + "\n" + + textual_edges.to_csv(index=False, columns=["src", "edge_attr", "dst"])) graph["desc"] = desc return graph + transform = apply_retrieval_via_pcst if args.use_pcst else apply_retrieval_with_text -query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 5}, seed_edges_kwargs={"k_edges": 5}, sampler_kwargs={"num_neighbors": [50]*2}, local_filter=transform) +query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 5}, + seed_edges_kwargs={"k_edges": 5}, + sampler_kwargs={"num_neighbors": [50] * 2}, + local_filter=transform) + # %% # Accuracy Metrics to be added to Profiler @@ -71,25 +94,32 @@ def _eidx_helper(subg: Data, ground_truth: Data): subg_e = set(subg_eidx) gt_e = set(gt_eidx) return subg_e, gt_e + + def check_retrieval_accuracy(subg: Data, ground_truth: Data, num_edges: int): subg_e, gt_e = _eidx_helper(subg, ground_truth) total_e = set(range(num_edges)) tp = len(subg_e & gt_e) - tn = len(total_e-(subg_e | gt_e)) - return (tp+tn)/num_edges + tn = len(total_e - (subg_e | gt_e)) + return (tp + tn) / num_edges + + def check_retrieval_precision(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(subg_e) + + def check_retrieval_recall(subg: Data, ground_truth: Data): subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) + # %% retrieval_stats = {"precision": [], "recall": [], "accuracy": []} subgs = [] node_len = [] edge_len = [] -for subg in tqdm.tqdm((query_loader.query(q) for q in questions)): +for subg in tqdm.tqdm(query_loader.query(q) for q in questions): subgs.append(subg) node_len.append(subg['x'].shape[0]) edge_len.append(subg['edge_attr'].shape[0]) @@ -98,6 +128,5 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subg['question'] = questions[i] subg['label'] = ds[i]['label'] -pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file+'_metadata.csv') -torch.save(subgs, args.out_file+'.pt') - +pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file + '_metadata.csv') +torch.save(subgs, args.out_file + '.pt') diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index 247913c7db92..a4e57e0e48a8 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -1,6 +1,7 @@ +import pytest + from torch_geometric.datasets import WebQSPDataset from torch_geometric.testing import onlyFullTest, onlyOnline -import pytest @pytest.mark.skip(reason="Times out") @@ -11,6 +12,7 @@ def test_web_qsp_dataset(): assert len(dataset) == 4700 assert str(dataset) == "WebQSPDataset(4700)" + @onlyOnline @onlyFullTest def test_web_qsp_dataset_limit(): @@ -18,6 +20,7 @@ def test_web_qsp_dataset_limit(): assert len(dataset) == 100 assert str(dataset) == "WebQSPDataset(100)" + @onlyOnline @onlyFullTest def test_web_qsp_dataset_limit_no_pcst(): diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index e29247203851..6adc66bbd662 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -23,18 +23,21 @@ def _setup_batch() -> Data: [list(range(10)), list(range(1, 10)) + [0]]) return batch + def _eval_train_call(model, batch): # test train loss = model(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.label, batch.edge_attr) assert loss is not None + def _eval_inference_call(model, batch): # test inference pred = model.inference(batch.question, batch.x, batch.edge_index, batch.batch, batch.ptr, batch.edge_attr) assert pred is not None + @onlyFullTest @withPackage('transformers', 'sentencepiece', 'accelerate') def test_g_retriever() -> None: @@ -46,10 +49,11 @@ def test_g_retriever() -> None: mlp_out_dim=2048, ) batch = batch.to(model.llm_device) - + _eval_train_call(model, batch) _eval_inference_call(model, batch) + @onlyFullTest @withPackage('transformers', 'sentencepiece', 'accelerate') def test_g_retriever_many_tokens() -> None: @@ -59,10 +63,8 @@ def test_g_retriever_many_tokens() -> None: llm_to_use="TinyLlama/TinyLlama-1.1B-Chat-v0.1", num_llm_params=1, # 1 Billion mlp_out_dim=2048, - mlp_out_tokens=2 - ) + mlp_out_tokens=2) batch = batch.to(model.llm_device) _eval_train_call(model, batch) _eval_inference_call(model, batch) - diff --git a/torch_geometric/data/large_graph_indexer.py b/torch_geometric/data/large_graph_indexer.py index 92f5cd492e78..51b0ce459b33 100644 --- a/torch_geometric/data/large_graph_indexer.py +++ b/torch_geometric/data/large_graph_indexer.py @@ -564,7 +564,8 @@ def __eq__(self, value: "LargeGraphIndexer") -> bool: def get_features_for_triplets_groups( indexer: LargeGraphIndexer, triplet_groups: Iterable[KnowledgeGraphLike], - node_feature_name: str = "x", edge_feature_name: str = "edge_attr", + node_feature_name: str = "x", + edge_feature_name: str = "edge_attr", pre_transform: Optional[Callable[[TripletLike], TripletLike]] = None, verbose: bool = False, ) -> Iterator[Data]: @@ -666,5 +667,6 @@ def get_features_for_triplets( """ gen = get_features_for_triplets_groups(indexer, [triplets], node_feature_name, - edge_feature_name, pre_transform, verbose) + edge_feature_name, pre_transform, + verbose) return next(gen) diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 7273b34d224d..91e4f502845c 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -1,5 +1,14 @@ from abc import abstractmethod -from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union, Iterator +from typing import ( + Any, + Callable, + Dict, + Iterator, + Optional, + Protocol, + Tuple, + Union, +) from torch_geometric.data import Data, FeatureStore, HeteroData from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput @@ -86,9 +95,9 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], self.sampler_kwargs = sampler_kwargs or {} self.loader_kwargs = loader_kwargs or {} - def batch_query(self, queries: Any, batch_size: int = 512) -> Iterator[Data]: + def batch_query(self, queries: Any, + batch_size: int = 512) -> Iterator[Data]: """Retrieves subraphs associated with each query in the batch (experimental).""" - for i in range(0, len(queries), batch_size): if i + batch_size <= len(queries): batch = queries[i:i + batch_size] @@ -100,12 +109,13 @@ def batch_query(self, queries: Any, batch_size: int = 512) -> Iterator[Data]: seed_edges = self.feature_store._retrieve_seed_edges_batch( batch, **self.seed_edges_kwargs) - for (seed_node, seed_edge, query) in zip(seed_nodes, seed_edges, batch): + for (seed_node, seed_edge, query) in zip(seed_nodes, seed_edges, + batch): subgraph_sample = self.graph_store.sample_subgraph( seed_node, seed_edge, **self.sampler_kwargs) - data = self.feature_store.load_subgraph(sample=subgraph_sample, - **self.loader_kwargs) + data = self.feature_store.load_subgraph( + sample=subgraph_sample, **self.loader_kwargs) if self.local_filter: data = self.local_filter(data, query) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index 1f4c151e62c3..72b047f10d7d 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -33,7 +33,7 @@ class GRetriever(nn.Module): :obj:`heads` kwarg. (default: :obj:`4`) mlp_hidden_dim (int): (default: :obj:`2048`) mlp_out_dim (int): (default: :obj:`4096`) - mlp_out_tokens (int) : Number of LLM prefix tokens to reserve for + mlp_out_tokens (int) : Number of LLM prefix tokens to reserve for GNN output. (default: :obj:`1`) .. warning:: @@ -118,7 +118,7 @@ def __init__( self.projector = nn.Sequential( nn.Linear(gnn_out_channels, mlp_hidden_dim), nn.Sigmoid(), - nn.Linear(mlp_hidden_dim, mlp_out_dim*mlp_out_tokens), + nn.Linear(mlp_hidden_dim, mlp_out_dim * mlp_out_tokens), nn.Unflatten(-1, (mlp_out_tokens, mlp_out_dim)), ).to(self.llm_device) @@ -225,10 +225,9 @@ def inference( num_nodes_per_graph = ptr[1:] - ptr[:-1] graph_embeds = self.encode_graphs(node_feat, edge_index, edge_attr, batch) - graph_embeds = [ - (embed if num_nodes_per_graph[i] != 0 else None) - for i, embed in enumerate(self.projector(graph_embeds)) - ] + graph_embeds = [(embed if num_nodes_per_graph[i] != 0 else None) + for i, embed in enumerate(self.projector(graph_embeds)) + ] inputs_embeds, attention_mask, _ = self.llm_to_use._get_embeds( question, additional_text_context, graph_embeds) with self.llm_to_use.autocast_context: From 5a9740080b1ca57518db3737643a76e814d58f1b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 06:11:34 -0700 Subject: [PATCH 703/752] precommit --- examples/llm_plus_gnn/doc/media/flowchart.svg | 1 - examples/llm_plus_gnn/doc/media/multihop_example.svg | 1 - examples/llm_plus_gnn/doc/media/remote_backend.svg | 1 - examples/llm_plus_gnn/multihop/multihop_preprocess.py | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 examples/llm_plus_gnn/doc/media/flowchart.svg delete mode 100644 examples/llm_plus_gnn/doc/media/multihop_example.svg delete mode 100644 examples/llm_plus_gnn/doc/media/remote_backend.svg diff --git a/examples/llm_plus_gnn/doc/media/flowchart.svg b/examples/llm_plus_gnn/doc/media/flowchart.svg deleted file mode 100644 index 188d37b14f41..000000000000 --- a/examples/llm_plus_gnn/doc/media/flowchart.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/examples/llm_plus_gnn/doc/media/multihop_example.svg b/examples/llm_plus_gnn/doc/media/multihop_example.svg deleted file mode 100644 index 42e2eaf36afa..000000000000 --- a/examples/llm_plus_gnn/doc/media/multihop_example.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/examples/llm_plus_gnn/doc/media/remote_backend.svg b/examples/llm_plus_gnn/doc/media/remote_backend.svg deleted file mode 100644 index c5791f0a95de..000000000000 --- a/examples/llm_plus_gnn/doc/media/remote_backend.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop/multihop_preprocess.py index 9d85c85f5034..80b3072c1501 100644 --- a/examples/llm_plus_gnn/multihop/multihop_preprocess.py +++ b/examples/llm_plus_gnn/multihop/multihop_preprocess.py @@ -1,4 +1,4 @@ -"""Example workflow for downloading and assembling a multihop QA dataset""" +"""Example workflow for downloading and assembling a multihop QA dataset.""" # %% [markdown] # # Encoding A Large Knowledge Graph Part 2 From fb57e264a00bfcdd125951679a328e85a3410a1d Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 06:15:21 -0700 Subject: [PATCH 704/752] ipython3 to python --- docs/source/advanced/rag.rst | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 595db8d22632..c8b6ce9c6dd0 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -71,7 +71,7 @@ retrieved is crucial for whether the appropriate response to the LLM. The same is true for GNN based RAG. For example, consider the WebQSPDataset. -.. code:: ipython3 +.. code:: python from torch_geometric.datasets import WebQSPDataset @@ -88,7 +88,7 @@ each entry in the dataset consists of: - A question to be answered - The answer - A knowledge graph subgraph of Freebase that has the context needed to answer the question. -.. code:: ipython3 +.. code:: python ds.raw_dataset @@ -104,7 +104,7 @@ needed to answer the question. -.. code:: ipython3 +.. code:: python ds.raw_dataset[0] @@ -139,7 +139,7 @@ the entries into a large knowledge graph, so that duplicate nodes and edges can be avoided, and so that alternative retrieval algorithms can be tried. We can do this with the LargeGraphIndexer class: -.. code:: ipython3 +.. code:: python from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups from torch_geometric.nn.nlp import SentenceTransformer @@ -149,7 +149,7 @@ be tried. We can do this with the LargeGraphIndexer class: from itertools import chain import networkx as nx -.. code:: ipython3 +.. code:: python raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']] print(raw_dataset_graphs[0][:10]) @@ -164,7 +164,7 @@ To show the benefits of this indexer in action, we will use the following model to encode this sample of graphs using LargeGraphIndexer, along with naively. -.. code:: ipython3 +.. code:: python device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) @@ -176,7 +176,7 @@ along with naively. First, we compare the clock times of encoding using both methods. -.. code:: ipython3 +.. code:: python # Indexing question-by-question dataset_graphs_embedded = [] @@ -229,7 +229,7 @@ First, we compare the clock times of encoding using both methods. -.. code:: ipython3 +.. code:: python # Using LargeGraphIndexer to make one large knowledge graph from torch_geometric.data.large_graph_indexer import EDGE_RELATION @@ -295,7 +295,7 @@ of the number of unique nodes and edges in the graph. We expect the two results to be functionally identical, with the differences being due to floating point jitter. -.. code:: ipython3 +.. code:: python def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8): def _sorted_tensors_are_close(tensor1, tensor2): @@ -306,7 +306,7 @@ differences being due to floating point jitter. and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \ and _graphs_are_same(ground_truth.edge_index, new_method.edge_index) -.. code:: ipython3 +.. code:: python all_results_match = True for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions): @@ -401,20 +401,20 @@ store the node and edge features in the graph. Let’s start by loading in a knowledge graph dataset for the sake of our experiment: -.. code:: ipython3 +.. code:: python from torch_geometric.data import LargeGraphIndexer from torch_geometric.datasets import WebQSPDataset from itertools import chain -.. code:: ipython3 +.. code:: python ds = WebQSPDataset(root='demo', limit=10) Let’s set up our set of questions and graph triplets: -.. code:: ipython3 +.. code:: python questions = ds.raw_dataset['question'] questions @@ -437,7 +437,7 @@ Let’s set up our set of questions and graph triplets: -.. code:: ipython3 +.. code:: python ds.raw_dataset[:10]['graph'][0][:10] @@ -461,7 +461,7 @@ Let’s set up our set of questions and graph triplets: -.. code:: ipython3 +.. code:: python all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset)) @@ -470,13 +470,13 @@ With these questions and triplets, we want to: 2. Create a FeatureStore that encodes all the nodes and edges in the knowledge graph 3. Create a GraphStore that encodes all the edge indices in the knowledge graph -.. code:: ipython3 +.. code:: python import torch from torch_geometric.nn.nlp import SentenceTransformer from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet -.. code:: ipython3 +.. code:: python import sys sys.path.append('..') @@ -485,7 +485,7 @@ In order to create a remote backend, we need to define a FeatureStore and GraphStore locally, as well as a method for initializing its state from triplets: -.. code:: ipython3 +.. code:: python from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader @@ -497,7 +497,7 @@ from triplets: # Ideally for a real remote backend, this interface would be replaced with an API to a vector DB, such as Pinecone. from rag_feature_store import SentenceTransformerFeatureStore -.. code:: ipython3 +.. code:: python device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = SentenceTransformer(model_name="sentence-transformers/all-roberta-large-v1").to(device) @@ -526,11 +526,11 @@ diagram: -.. code:: ipython3 +.. code:: python from torch_geometric.loader import RAGQueryLoader -.. code:: ipython3 +.. code:: python query_loader = RAGQueryLoader( data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store @@ -568,11 +568,11 @@ unique hyperparameters for each unique step in this retrieval. Now we can submit our queries to the remote backend to retrieve our subgraphs: -.. code:: ipython3 +.. code:: python import tqdm -.. code:: ipython3 +.. code:: python sub_graphs = [] for q in tqdm.tqdm(questions): @@ -585,7 +585,7 @@ subgraphs: 100%|██████████| 10/10 [00:07<00:00, 1.28it/s] -.. code:: ipython3 +.. code:: python sub_graphs[0] @@ -603,11 +603,11 @@ when compared to the original WebQSP dataset. Can we compare the properties of this method to the original WebQSPDataset’s retrieval method? Let’s compare some basics properties of the subgraphs: -.. code:: ipython3 +.. code:: python from torch_geometric.data import Data -.. code:: ipython3 +.. code:: python def _eidx_helper(subg: Data, ground_truth: Data): subg_eidx, gt_eidx = subg.edge_idx, ground_truth.edge_idx @@ -631,16 +631,16 @@ method? Let’s compare some basics properties of the subgraphs: subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) -.. code:: ipython3 +.. code:: python from torch_geometric.data import get_features_for_triplets_groups -.. code:: ipython3 +.. code:: python ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) num_edges = len(ds.indexer._edges) -.. code:: ipython3 +.. code:: python for subg, ground_truth in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): print(f"Size: {len(subg.x)}, Ground Truth Size: {len(ground_truth.x)}, Accuracy: {check_retrieval_accuracy(subg, ground_truth, num_edges)}, Precision: {check_retrieval_precision(subg, ground_truth)}, Recall: {check_retrieval_recall(subg, ground_truth)}") From 30fe562829f7cd8321f3c6b4d3a5f74958f8c37a Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 06:19:51 -0700 Subject: [PATCH 705/752] remove batch query for now --- torch_geometric/loader/rag_loader.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 91e4f502845c..510d82713ecc 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -95,32 +95,6 @@ def __init__(self, data: Tuple[RAGFeatureStore, RAGGraphStore], self.sampler_kwargs = sampler_kwargs or {} self.loader_kwargs = loader_kwargs or {} - def batch_query(self, queries: Any, - batch_size: int = 512) -> Iterator[Data]: - """Retrieves subraphs associated with each query in the batch (experimental).""" - for i in range(0, len(queries), batch_size): - if i + batch_size <= len(queries): - batch = queries[i:i + batch_size] - else: - batch = queries[i:] - - seed_nodes = self.feature_store._retrieve_seed_nodes_batch( - batch, **self.seed_nodes_kwargs) - seed_edges = self.feature_store._retrieve_seed_edges_batch( - batch, **self.seed_edges_kwargs) - - for (seed_node, seed_edge, query) in zip(seed_nodes, seed_edges, - batch): - subgraph_sample = self.graph_store.sample_subgraph( - seed_node, seed_edge, **self.sampler_kwargs) - - data = self.feature_store.load_subgraph( - sample=subgraph_sample, **self.loader_kwargs) - - if self.local_filter: - data = self.local_filter(data, query) - yield data - def query(self, query: Any) -> Data: """Retrieve a subgraph associated with the query with all its feature attributes. From 27843dccf169ab66a7cfca111b47f19fb48d8e8f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 06:43:33 -0700 Subject: [PATCH 706/752] precommit 2 --- .../llm_plus_gnn/benchmark_model_archs.py | 4 +- examples/llm_plus_gnn/g_retriever.py | 4 +- .../multihop/multihop_preprocess.py | 93 +++++++++++++------ .../multihop/rag_generate_multihop.py | 20 ++-- .../nvtx_examples/nvtx_rag_backend_example.py | 15 ++- examples/llm_plus_gnn/rag_feature_store.py | 4 +- examples/llm_plus_gnn/rag_generate.py | 3 +- torch_geometric/loader/rag_loader.py | 11 +-- 8 files changed, 98 insertions(+), 56 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs.py index 1a4184344cab..7737083b114a 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs.py +++ b/examples/llm_plus_gnn/benchmark_model_archs.py @@ -43,7 +43,9 @@ dataset = torch.load(args.dataset_path) class MockDataset: - """Utility class to patch the fields in WebQSPDataset used by GRetriever.""" + """Utility class to patch the fields in WebQSPDataset used by + GRetriever. + """ def __init__(self) -> None: pass diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 1944a9ef33fa..3585debb32e1 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -625,8 +625,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, final_prnt_str += "Prompt: '" + question[j] + "'\n" final_prnt_str += "Label: '" + correct_answer[j] + "'\n" if not skip_pretrained_LLM: - final_prnt_str += "Untuned LLM Output: '" + untuned_llm_pred[ - j] + "'\n" # noqa + final_prnt_str += "Untuned LLM Output: '" \ + + untuned_llm_pred[j] + "'\n" # noqa final_prnt_str += "Tuned LLM Output: '" + pure_llm_pred[ j] + "'\n" final_prnt_str += "GNN+LLM Output: '" + gnn_llm_pred[j] + "'\n" diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop/multihop_preprocess.py index 80b3072c1501..46052bdf1b15 100644 --- a/examples/llm_plus_gnn/multihop/multihop_preprocess.py +++ b/examples/llm_plus_gnn/multihop/multihop_preprocess.py @@ -1,45 +1,65 @@ """Example workflow for downloading and assembling a multihop QA dataset.""" +import argparse +import json +from subprocess import call + +import pandas as pd +import torch +import tqdm + +from torch_geometric.data import LargeGraphIndexer + # %% [markdown] # # Encoding A Large Knowledge Graph Part 2 # %% [markdown] -# In this notebook, we will continue where we left off by building a new multi-hop QA dataset based on Wikidata. +# In this notebook, we will continue where we left off by building a new +# multi-hop QA dataset based on Wikidata. # %% [markdown] -# ## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph +# ## Example 2: Building a new Dataset from Questions and an already-existing +# Knowledge Graph # %% [markdown] # ### Motivation # %% [markdown] -# One potential application of knowledge graph structural encodings is capturing the relationships between different entities that are multiple hops apart. This can be challenging for an LLM to recognize from prepended graph information. Here's a motivating example (credit to @Rishi Puri): +# One potential application of knowledge graph structural encodings is +# capturing the relationships between different entities that are multiple +# hops apart. This can be challenging for an LLM to recognize from prepended +# graph information. Here's a motivating example (credit to @Rishi Puri): # %% [markdown] -# In this example, the question can only be answered by reasoning about the relationships between the entities in the knowledge graph. +# In this example, the question can only be answered by reasoning about the +# relationships between the entities in the knowledge graph. # %% [markdown] # ### Building a Multi-Hop QA Dataset # %% [markdown] -# To start, we need to download the raw data of a knowledge graph. In this case, we use WikiData5M ([Wang et al](https://paperswithcode.com/paper/kepler-a-unified-model-for-knowledge)). Here we download the raw triplets and their entity codes. Information about this dataset can be found [here](https://deepgraphlearning.github.io/project/wikidata5m). +# To start, we need to download the raw data of a knowledge graph. +# In this case, we use WikiData5M +# ([Wang et al] +# (https://paperswithcode.com/paper/kepler-a-unified-model-for-knowledge)). +# Here we download the raw triplets and their entity codes. Information about +# this dataset can be found +# [here](https://deepgraphlearning.github.io/project/wikidata5m). # %% [markdown] -# The following download contains the ID to plaintext mapping for all the entities and relations in the knowledge graph: - -from subprocess import call +# The following download contains the ID to plaintext mapping for all the +# entities and relations in the knowledge graph: rv = call("./multihop_download.sh") # %% [markdown] -# To start, we are going to preprocess the knowledge graph to substitute each of the entity/relation codes with their plaintext aliases. This makes it easier to use a pre-trained textual encoding model to create triplet embeddings, as such a model likely won't understand how to properly embed the entity codes. - -import argparse -import json +# To start, we are going to preprocess the knowledge graph to substitute each +# of the entity/relation codes with their plaintext aliases. This makes it +# easier to use a pre-trained textual encoding model to create triplet +# embeddings, as such a model likely won't understand how to properly embed +# the entity codes. # %% -import pandas as pd -import tqdm # %% parser = argparse.ArgumentParser(description="Preprocess wikidata5m") @@ -89,10 +109,18 @@ print(f"Missing aliases: {missing_total}/{total}") # %% [markdown] -# Now `full_graph` represents the knowledge graph triplets in understandable plaintext. +# Now `full_graph` represents the knowledge graph triplets in +# understandable plaintext. # %% [markdown] -# Next, we need a set of multi-hop questions that the Knowledge Graph will provide us with context for. We utilize a subset of [HotPotQA](https://hotpotqa.github.io/) ([Yang et. al.](https://arxiv.org/pdf/1809.09600)) called [2WikiMultiHopQA](https://github.com/Alab-NII/2wikimultihop) ([Ho et. al.](https://aclanthology.org/2020.coling-main.580.pdf)), which includes a subgraph of entities that serve as the ground truth justification for answering each multi-hop question: +# Next, we need a set of multi-hop questions that the Knowledge Graph will +# provide us with context for. We utilize a subset of +# [HotPotQA](https://hotpotqa.github.io/) +# ([Yang et. al.](https://arxiv.org/pdf/1809.09600)) called +# [2WikiMultiHopQA](https://github.com/Alab-NII/2wikimultihop) +# ([Ho et. al.](https://aclanthology.org/2020.coling-main.580.pdf)), +# which includes a subgraph of entities that serve as the ground truth +# justification for answering each multi-hop question: # %% with open('train.json') as f: @@ -119,7 +147,8 @@ df['graph_size'] = df['evidences_id'].apply(lambda row: len(row)) # %% [markdown] -# (Optional) We take only questions where the evidence graph is greater than 0. (Note: this gets rid of the test set): +# (Optional) We take only questions where the evidence graph is greater than +# 0. (Note: this gets rid of the test set): # %% # df = df[df['graph_size'] > 0] @@ -137,7 +166,8 @@ refined_df.to_csv('wikimultihopqa_refined.csv', index=False) # %% [markdown] -# Now we need to check that all the entities mentioned in the question/answer set are also present in the Wikidata graph: +# Now we need to check that all the entities mentioned in the question/answer +# set are also present in the Wikidata graph: # %% relation_map = {} @@ -163,15 +193,20 @@ entities = trip[0], trip[2] for entity in entities: if entity not in entity_set: - # print(f'The following entity was not found in the KG: {entity}') + # print( + # f'The following entity was not found in the KG: {entity}' + # ) missing_entities.add(entity) missing_entity_idx.add(i) # %% [markdown] -# Right now, we drop the missing entity entries. Additional preprocessing can be done here to resolve the entity/relation collisions, but that is out of the scope for this notebook. +# Right now, we drop the missing entity entries. Additional preprocessing can +# be done here to resolve the entity/relation collisions, but that is out of +# the scope for this notebook. # %% -# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped. +# missing relations are ok, but missing entities cannot be mapped to +# plaintext, so they should be dropped. refined_df.reset_index(inplace=True, drop=True) # %% @@ -184,7 +219,6 @@ cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False) # %% -import torch # %% torch.save(full_graph, 'wikimultihopqa_full_graph.pt') @@ -193,16 +227,22 @@ # ### Question: How do we extract a contextual subgraph for a given query? # %% [markdown] -# The chosen retrieval algorithm is a critical component in the pipeline for affecting RAG performance. In the next section (1), we will demonstrate a naive method of retrieval for a large knowledge graph, and how to apply it to this dataset along with WebQSP. +# The chosen retrieval algorithm is a critical component in the pipeline for +# affecting RAG performance. In the next section (1), we will demonstrate a +# naive method of retrieval for a large knowledge graph, and how to apply it +# to this dataset along with WebQSP. # %% [markdown] # ### Preparing a Textualized Graph for LLM # %% [markdown] -# For now however, we need to prepare the graph data to be used as a plaintext prefix to the LLM. In order to do this, we want to prompt the LLM to use the unique nodes, and unique edge triplets of a given subgraph. In order to do this, we prepare a unique indexed node df and edge df for the knowledge graph now. This process occurs trivially with the LargeGraphIndexer: +# For now however, we need to prepare the graph data to be used as a plaintext +# prefix to the LLM. In order to do this, we want to prompt the LLM to use the +# unique nodes, and unique edge triplets of a given subgraph. In order to do +# this, we prepare a unique indexed node df and edge df for the knowledge +# graph now. This process occurs trivially with the LargeGraphIndexer: # %% -from torch_geometric.data import LargeGraphIndexer # %% indexer = LargeGraphIndexer.from_triplets(full_graph) @@ -225,7 +265,8 @@ textual_edges["dst"] = [indexer._nodes[h] for h in textual_edges["dst"]] # %% [markdown] -# Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph. +# Note: The edge table refers to each node by its index in the node table. +# We will see how this gets utilized later when indexing a subgraph. # %% [markdown] # Now we can save the result diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index a0419deff36d..eb06fb833166 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -1,17 +1,13 @@ # %% +import argparse import sys +from typing import Tuple import pandas as pd import torch import tqdm -sys.path.append('..') -import argparse - -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerApproxFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore - +from torch_geometric.data import Data from torch_geometric.datasets.web_qsp_dataset import ( preprocess_triplet, retrieval_via_pcst, @@ -19,6 +15,13 @@ from torch_geometric.loader import RAGQueryLoader from torch_geometric.nn.nlp import SentenceTransformer +sys.path.append('..') + +from profiling_utils import create_remote_backend_from_triplets # noqa: E402 +from rag_feature_store import \ + SentenceTransformerApproxFeatureStore # noqa: E402 +from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 + # %% parser = argparse.ArgumentParser( description="Generate new multihop dataset for rag") @@ -48,9 +51,6 @@ feature_db=SentenceTransformerApproxFeatureStore).load() # %% -from typing import Tuple - -from torch_geometric.data import Data all_textual_nodes = pd.read_csv('wikimultihopqa_textual_nodes.csv') all_textual_edges = pd.read_csv('wikimultihopqa_textual_edges.csv') diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index 69b1602e4316..c341877dc480 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -1,20 +1,26 @@ # %% import argparse +import sys from itertools import chain from typing import Tuple import torch -from profiling_utils import create_remote_backend_from_triplets -from rag_feature_store import SentenceTransformerFeatureStore -from rag_graph_store import NeighborSamplingRAGGraphStore from torch_geometric.data import Data, get_features_for_triplets_groups from torch_geometric.datasets import WebQSPDataset -from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet +from torch_geometric.datasets.web_qsp_dataset import ( + preprocess_triplet, + retrieval_via_pcst, +) from torch_geometric.loader import rag_loader from torch_geometric.nn.nlp import SentenceTransformer from torch_geometric.profile.nvtx import nvtxit +sys.path.append('..') +from profiling_utils import create_remote_backend_from_triplets # noqa: E402 +from rag_feature_store import SentenceTransformerFeatureStore # noqa: E402 +from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 + # %% # Patch FeatureStore and GraphStore @@ -57,7 +63,6 @@ feature_db=SentenceTransformerFeatureStore).load() # %% -from torch_geometric.datasets.web_qsp_dataset import retrieval_via_pcst @nvtxit() diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/rag_feature_store.py index b9b60a0070c6..e01e9e59bb88 100644 --- a/examples/llm_plus_gnn/rag_feature_store.py +++ b/examples/llm_plus_gnn/rag_feature_store.py @@ -106,7 +106,9 @@ def _add_features_to_knn_index(knn_index: ApproxMIPSKNNIndex, emb: Tensor, knn_index (ApproxMIPSKNNIndex): Index to add features to. emb (Tensor): Embeddings to add. device (torch.device): Device to store in - batch_size (int, optional): Batch size to iterate by. Defaults to 2**20, which equates to 4GB if working with 1024 dim floats. + batch_size (int, optional): Batch size to iterate by. + Defaults to 2**20, which equates to 4GB if working with + 1024 dim floats. """ for i in range(0, emb.size(0), batch_size): if emb.size(0) - i >= batch_size: diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index e75eb3b311b2..14b116e5424f 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -75,7 +75,8 @@ def apply_retrieval_with_text(graph: Data, query: str) -> Tuple[Data, str]: return graph -transform = apply_retrieval_via_pcst if args.use_pcst else apply_retrieval_with_text +transform = apply_retrieval_via_pcst \ + if args.use_pcst else apply_retrieval_with_text query_loader = RAGQueryLoader(data=(fs, gs), seed_nodes_kwargs={"k_nodes": 5}, seed_edges_kwargs={"k_edges": 5}, diff --git a/torch_geometric/loader/rag_loader.py b/torch_geometric/loader/rag_loader.py index 510d82713ecc..33d6cf0e868e 100644 --- a/torch_geometric/loader/rag_loader.py +++ b/torch_geometric/loader/rag_loader.py @@ -1,14 +1,5 @@ from abc import abstractmethod -from typing import ( - Any, - Callable, - Dict, - Iterator, - Optional, - Protocol, - Tuple, - Union, -) +from typing import Any, Callable, Dict, Optional, Protocol, Tuple, Union from torch_geometric.data import Data, FeatureStore, HeteroData from torch_geometric.sampler import HeteroSamplerOutput, SamplerOutput From 204ac8f43712b0ecafa9563a1986b1960d28c252 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 07:05:08 -0700 Subject: [PATCH 707/752] clean up code in docs 1 --- docs/source/advanced/rag.rst | 242 +++++------------------------------ 1 file changed, 32 insertions(+), 210 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index c8b6ce9c6dd0..3be36c491229 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -84,20 +84,17 @@ Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So -each entry in the dataset consists of: - A question to be answered - The -answer - A knowledge graph subgraph of Freebase that has the context +each entry in the dataset consists of: +- A question to be answered +- The answer +- A knowledge graph subgraph of Freebase that has the context needed to answer the question. .. code:: python ds.raw_dataset - - - -.. parsed-literal:: - - Dataset({ + >>> Dataset({ features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'], num_rows: 100 }) @@ -109,11 +106,7 @@ needed to answer the question. ds.raw_dataset[0] - - -.. parsed-literal:: - - {'id': 'WebQTrn-0', + >>> {'id': 'WebQTrn-0', 'question': 'what is the name of justin bieber brother', 'answer': ['Jaxon Bieber'], 'q_entity': ['Justin Bieber'], @@ -130,7 +123,7 @@ from doing so: 1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond to the algorithm that was used to generate the dataset subgraphs. -2. The dataset as is not stored computationally efficiently, as there will +1. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions. @@ -154,10 +147,7 @@ be tried. We can do this with the LargeGraphIndexer class: raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']] print(raw_dataset_graphs[0][:10]) - -.. parsed-literal:: - - [('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell')] + >>> [('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ...] To show the benefits of this indexer in action, we will use the @@ -169,10 +159,6 @@ along with naively. device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device) -.. parsed-literal:: - - cuda - First, we compare the clock times of encoding using both methods. @@ -218,14 +204,8 @@ First, we compare the clock times of encoding using both methods. print(time.time()-start) - -.. parsed-literal:: - - 100%|██████████| 100/100 [02:01<00:00, 1.22s/it] - -.. parsed-literal:: - - 121.68579435348511 + >>> 100%|██████████| 100/100 [02:01<00:00, 1.22s/it] + >>> 121.68579435348511 @@ -263,22 +243,10 @@ First, we compare the clock times of encoding using both methods. dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)] print(time.time()-start-whole_graph_diff) - -.. parsed-literal:: - - Indexing... - Time to create whole knowledge_graph: 114.01080107688904 - Retrieving Subgraphs... - - -.. parsed-literal:: - - 100%|██████████| 100/100 [00:00<00:00, 212.87it/s] - 100%|██████████| 100/100 [00:01<00:00, 80.90it/s] - -.. parsed-literal:: - - 114.66037964820862 + >>> Indexing... + >>> Time to create whole knowledge_graph: 114.01080107688904 + >>> Retrieving Subgraphs... + >>> 114.66037964820862 The large graph indexer allows us to compute the entire knowledge graph @@ -306,24 +274,13 @@ differences being due to floating point jitter. and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \ and _graphs_are_same(ground_truth.edge_index, new_method.edge_index) -.. code:: python all_results_match = True for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions): all_results_match &= results_are_close_enough(old_graph, new_graph) all_results_match - -.. parsed-literal:: - - 100%|██████████| 100/100 [00:25<00:00, 4.00it/s] - - - - -.. parsed-literal:: - - True + >>> True @@ -407,9 +364,6 @@ experiment: from torch_geometric.datasets import WebQSPDataset from itertools import chain - -.. code:: python - ds = WebQSPDataset(root='demo', limit=10) Let’s set up our set of questions and graph triplets: @@ -419,12 +373,7 @@ Let’s set up our set of questions and graph triplets: questions = ds.raw_dataset['question'] questions - - - -.. parsed-literal:: - - ['what is the name of justin bieber brother', + >>> ['what is the name of justin bieber brother', 'what character did natalie portman play in star wars', 'what country is the grand bahama island in', 'what kind of money to take to bahamas', @@ -436,17 +385,10 @@ Let’s set up our set of questions and graph triplets: 'which countries border the us'] - -.. code:: python - ds.raw_dataset[:10]['graph'][0][:10] - - -.. parsed-literal:: - - [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'], + >>> [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'], ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'], ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'], ['Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'], @@ -460,9 +402,6 @@ Let’s set up our set of questions and graph triplets: ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell']] - -.. code:: python - all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset)) With these questions and triplets, we want to: @@ -470,23 +409,14 @@ With these questions and triplets, we want to: 2. Create a FeatureStore that encodes all the nodes and edges in the knowledge graph 3. Create a GraphStore that encodes all the edge indices in the knowledge graph -.. code:: python - - import torch - from torch_geometric.nn.nlp import SentenceTransformer - from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet - -.. code:: python - - import sys - sys.path.append('..') - In order to create a remote backend, we need to define a FeatureStore and GraphStore locally, as well as a method for initializing its state -from triplets: +from triplets. The code methods used in this tutorial can be found in +`examples/llm_plus_gnn`. .. code:: python + from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader # We define this GraphStore to sample the neighbors of a node locally. @@ -530,8 +460,6 @@ diagram: from torch_geometric.loader import RAGQueryLoader -.. code:: python - query_loader = RAGQueryLoader( data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store # Arguments to pass into the seed node/edge retrieval methods for the FeatureStore. @@ -568,33 +496,17 @@ unique hyperparameters for each unique step in this retrieval. Now we can submit our queries to the remote backend to retrieve our subgraphs: -.. code:: python - - import tqdm - .. code:: python sub_graphs = [] for q in tqdm.tqdm(questions): sub_graphs.append(query_loader.query(q)) - - -.. parsed-literal:: - - 100%|██████████| 10/10 [00:07<00:00, 1.28it/s] - - -.. code:: python + >>> 100%|██████████| 10/10 [00:07<00:00, 1.28it/s] sub_graphs[0] - - - -.. parsed-literal:: - - Data(x=[2251, 1024], edge_index=[2, 7806], edge_attr=[7806, 1024], node_idx=[2251], edge_idx=[7806]) + >>> Data(x=[2251, 1024], edge_index=[2, 7806], edge_attr=[7806, 1024], node_idx=[2251], edge_idx=[7806]) @@ -603,10 +515,6 @@ when compared to the original WebQSP dataset. Can we compare the properties of this method to the original WebQSPDataset’s retrieval method? Let’s compare some basics properties of the subgraphs: -.. code:: python - - from torch_geometric.data import Data - .. code:: python def _eidx_helper(subg: Data, ground_truth: Data): @@ -631,110 +539,24 @@ method? Let’s compare some basics properties of the subgraphs: subg_e, gt_e = _eidx_helper(subg, ground_truth) return len(subg_e & gt_e) / len(gt_e) -.. code:: python - - from torch_geometric.data import get_features_for_triplets_groups - -.. code:: python ground_truth_graphs = get_features_for_triplets_groups(ds.indexer, (d['graph'] for d in ds.raw_dataset), pre_transform=preprocess_triplet) num_edges = len(ds.indexer._edges) -.. code:: python for subg, ground_truth in tqdm.tqdm(zip((query_loader.query(q) for q in questions), ground_truth_graphs)): print(f"Size: {len(subg.x)}, Ground Truth Size: {len(ground_truth.x)}, Accuracy: {check_retrieval_accuracy(subg, ground_truth, num_edges)}, Precision: {check_retrieval_precision(subg, ground_truth)}, Recall: {check_retrieval_recall(subg, ground_truth)}") - -.. parsed-literal:: - - 10it [00:00, 60.20it/s] - 1it [00:00, 1.18it/s] - -.. parsed-literal:: - - Size: 2193, Ground Truth Size: 1709, Accuracy: 0.6636780705203827, Precision: 0.22923807012918535, Recall: 0.1994037381034285 - - -.. parsed-literal:: - - 2it [00:01, 1.41it/s] - -.. parsed-literal:: - - Size: 2682, Ground Truth Size: 1251, Accuracy: 0.7158736400576746, Precision: 0.10843513670738801, Recall: 0.22692963233503774 - - -.. parsed-literal:: - - 3it [00:02, 1.51it/s] - -.. parsed-literal:: - - Size: 2087, Ground Truth Size: 1285, Accuracy: 0.7979813868134749, Precision: 0.0547879177377892, Recall: 0.15757855822550831 - - -.. parsed-literal:: - - 4it [00:02, 1.56it/s] - -.. parsed-literal:: - - Size: 2975, Ground Truth Size: 1988, Accuracy: 0.6956088609254162, Precision: 0.14820555621795636, Recall: 0.21768826619964973 - - -.. parsed-literal:: - - 5it [00:03, 1.59it/s] - -.. parsed-literal:: - - Size: 2594, Ground Truth Size: 633, Accuracy: 0.78849128326124, Precision: 0.04202616198163095, Recall: 0.2032301480484522 - - -.. parsed-literal:: - - 6it [00:03, 1.61it/s] - -.. parsed-literal:: - - Size: 2462, Ground Truth Size: 1044, Accuracy: 0.7703499803381832, Precision: 0.07646643109540636, Recall: 0.19551861221539574 - - -.. parsed-literal:: - - 7it [00:04, 1.62it/s] - -.. parsed-literal:: - - Size: 2011, Ground Truth Size: 1382, Accuracy: 0.7871804954777821, Precision: 0.10117783355860205, Recall: 0.13142713819914723 - - -.. parsed-literal:: - - 8it [00:05, 1.63it/s] - -.. parsed-literal:: - - Size: 2011, Ground Truth Size: 1052, Accuracy: 0.802831301612269, Precision: 0.06452691407556001, Recall: 0.16702726092600606 - - -.. parsed-literal:: - - 9it [00:05, 1.64it/s] - -.. parsed-literal:: - - Size: 2892, Ground Truth Size: 1012, Accuracy: 0.7276182985974571, Precision: 0.10108615156751419, Recall: 0.20860927152317882 - - -.. parsed-literal:: - - 10it [00:06, 1.58it/s] - -.. parsed-literal:: - - Size: 1817, Ground Truth Size: 1978, Accuracy: 0.7530475815965395, Precision: 0.1677807486631016, Recall: 0.11696178937558248 + >>> Size: 2193, Ground Truth Size: 1709, Accuracy: 0.6636780705203827, Precision: 0.22923807012918535, Recall: 0.1994037381034285 + >>> Size: 2682, Ground Truth Size: 1251, Accuracy: 0.7158736400576746, Precision: 0.10843513670738801, Recall: 0.22692963233503774 + >>> Size: 2087, Ground Truth Size: 1285, Accuracy: 0.7979813868134749, Precision: 0.0547879177377892, Recall: 0.15757855822550831 + >>> Size: 2975, Ground Truth Size: 1988, Accuracy: 0.6956088609254162, Precision: 0.14820555621795636, Recall: 0.21768826619964973 + >>> Size: 2594, Ground Truth Size: 633, Accuracy: 0.78849128326124, Precision: 0.04202616198163095, Recall: 0.2032301480484522 + >>> Size: 2462, Ground Truth Size: 1044, Accuracy: 0.7703499803381832, Precision: 0.07646643109540636, Recall: 0.19551861221539574 + >>> Size: 2011, Ground Truth Size: 1382, Accuracy: 0.7871804954777821, Precision: 0.10117783355860205, Recall: 0.13142713819914723 + >>> Size: 2011, Ground Truth Size: 1052, Accuracy: 0.802831301612269, Precision: 0.06452691407556001, Recall: 0.16702726092600606 + >>> Size: 2892, Ground Truth Size: 1012, Accuracy: 0.7276182985974571, Precision: 0.10108615156751419, Recall: 0.20860927152317882 + >>> Size: 1817, Ground Truth Size: 1978, Accuracy: 0.7530475815965395, Precision: 0.1677807486631016, Recall: 0.11696178937558248 From 3c05386995a9bd6adca13486daf17146d61190ff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:12:09 +0000 Subject: [PATCH 708/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/advanced/rag.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 3be36c491229..32daf2fa6630 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -85,9 +85,9 @@ maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So each entry in the dataset consists of: -- A question to be answered +- A question to be answered - The answer -- A knowledge graph subgraph of Freebase that has the context +- A knowledge graph subgraph of Freebase that has the context needed to answer the question. .. code:: python @@ -411,7 +411,7 @@ With these questions and triplets, we want to: In order to create a remote backend, we need to define a FeatureStore and GraphStore locally, as well as a method for initializing its state -from triplets. The code methods used in this tutorial can be found in +from triplets. The code methods used in this tutorial can be found in `examples/llm_plus_gnn`. .. code:: python From 20fc46dcf42ff0ecb1c7a14aaf3cde98cfa1a3f5 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 07:17:42 -0700 Subject: [PATCH 709/752] remove tqdm bars --- docs/source/advanced/rag.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 32daf2fa6630..221c42f38932 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -204,7 +204,6 @@ First, we compare the clock times of encoding using both methods. print(time.time()-start) - >>> 100%|██████████| 100/100 [02:01<00:00, 1.22s/it] >>> 121.68579435348511 @@ -502,7 +501,6 @@ subgraphs: for q in tqdm.tqdm(questions): sub_graphs.append(query_loader.query(q)) - >>> 100%|██████████| 10/10 [00:07<00:00, 1.28it/s] sub_graphs[0] From 1a33f5a5fe44514eade2fab719513e8a5e31647c Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 08:23:02 -0700 Subject: [PATCH 710/752] nvtx unit tests --- test/profile/test_nvtx.py | 122 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 test/profile/test_nvtx.py diff --git a/test/profile/test_nvtx.py b/test/profile/test_nvtx.py new file mode 100644 index 000000000000..4bc335d2fe78 --- /dev/null +++ b/test/profile/test_nvtx.py @@ -0,0 +1,122 @@ +from unittest.mock import patch, call +from torch_geometric.profile import nvtxit + +def _setup_mock(torch_cuda_mock): + torch_cuda_mock.is_available.return_value = True + torch_cuda_mock.cudart.return_value.cudaProfilerStart.return_value = None + torch_cuda_mock.cudart.return_value.cudaProfilerStop.return_value = None + return torch_cuda_mock + +@patch('torch_geometric.profile.nvtx.torch.cuda') +def test_nvtxit_base(torch_cuda_mock): + torch_cuda_mock = _setup_mock(torch_cuda_mock) + + # dummy func calls a calls b + + @nvtxit() + def call_b(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return 42 + + @nvtxit() + def call_a(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return call_b() + + def dummy_func(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return call_a() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + dummy_func() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_a_0'), call('call_b_0')] + +@patch('torch_geometric.profile.nvtx.torch.cuda') +def test_nvtxit_rename(torch_cuda_mock): + torch_cuda_mock = _setup_mock(torch_cuda_mock) + + # dummy func calls a calls b + + @nvtxit() + def call_b(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return 42 + + @nvtxit('a_nvtx') + def call_a(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return call_b() + + def dummy_func(): + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + return call_a() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + dummy_func() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('a_nvtx_0'), call('call_b_0')] + +@patch('torch_geometric.profile.nvtx.torch.cuda') +def test_nvtxit_iters(torch_cuda_mock): + torch_cuda_mock = _setup_mock(torch_cuda_mock) + + # dummy func calls a calls b + + @nvtxit(n_iters=1) + def call_b(): + return 42 + + @nvtxit() + def call_a(): + return call_b() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + + call_b() + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + call_a() + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 2 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 2 + + assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_b_0'), call('call_a_0')] + +@patch('torch_geometric.profile.nvtx.torch.cuda') +def test_nvtxit_warmups(torch_cuda_mock): + torch_cuda_mock = _setup_mock(torch_cuda_mock) + + # dummy func calls a calls b + + @nvtxit(n_warmups=1) + def call_b(): + return 42 + + @nvtxit() + def call_a(): + return call_b() + + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + + call_b() + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + call_a() + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + + assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_a_0'), call('call_b_1')] From 9aa42449066f0a2046bdc200c518c473be38cc10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:25:29 +0000 Subject: [PATCH 711/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/profile/test_nvtx.py | 40 ++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/test/profile/test_nvtx.py b/test/profile/test_nvtx.py index 4bc335d2fe78..ffe193e733cf 100644 --- a/test/profile/test_nvtx.py +++ b/test/profile/test_nvtx.py @@ -1,12 +1,15 @@ -from unittest.mock import patch, call +from unittest.mock import call, patch + from torch_geometric.profile import nvtxit + def _setup_mock(torch_cuda_mock): torch_cuda_mock.is_available.return_value = True torch_cuda_mock.cudart.return_value.cudaProfilerStart.return_value = None torch_cuda_mock.cudart.return_value.cudaProfilerStop.return_value = None return torch_cuda_mock + @patch('torch_geometric.profile.nvtx.torch.cuda') def test_nvtxit_base(torch_cuda_mock): torch_cuda_mock = _setup_mock(torch_cuda_mock) @@ -18,7 +21,7 @@ def call_b(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 return 42 - + @nvtxit() def call_a(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 @@ -29,14 +32,17 @@ def dummy_func(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 return call_a() - + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 dummy_func() assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 - assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_a_0'), call('call_b_0')] + assert torch_cuda_mock.nvtx.range_push.call_args_list == [ + call('call_a_0'), call('call_b_0') + ] + @patch('torch_geometric.profile.nvtx.torch.cuda') def test_nvtxit_rename(torch_cuda_mock): @@ -49,7 +55,7 @@ def call_b(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 return 42 - + @nvtxit('a_nvtx') def call_a(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 @@ -60,14 +66,17 @@ def dummy_func(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 return call_a() - + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 dummy_func() assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 - assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('a_nvtx_0'), call('call_b_0')] + assert torch_cuda_mock.nvtx.range_push.call_args_list == [ + call('a_nvtx_0'), call('call_b_0') + ] + @patch('torch_geometric.profile.nvtx.torch.cuda') def test_nvtxit_iters(torch_cuda_mock): @@ -78,11 +87,11 @@ def test_nvtxit_iters(torch_cuda_mock): @nvtxit(n_iters=1) def call_b(): return 42 - + @nvtxit() def call_a(): return call_b() - + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 @@ -93,7 +102,10 @@ def call_a(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 2 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 2 - assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_b_0'), call('call_a_0')] + assert torch_cuda_mock.nvtx.range_push.call_args_list == [ + call('call_b_0'), call('call_a_0') + ] + @patch('torch_geometric.profile.nvtx.torch.cuda') def test_nvtxit_warmups(torch_cuda_mock): @@ -104,11 +116,11 @@ def test_nvtxit_warmups(torch_cuda_mock): @nvtxit(n_warmups=1) def call_b(): return 42 - + @nvtxit() def call_a(): return call_b() - + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 @@ -119,4 +131,6 @@ def call_a(): assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 - assert torch_cuda_mock.nvtx.range_push.call_args_list == [call('call_a_0'), call('call_b_1')] + assert torch_cuda_mock.nvtx.range_push.call_args_list == [ + call('call_a_0'), call('call_b_1') + ] From 5915b3d290217912b3351140bcb50e1254688454 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 08:37:42 -0700 Subject: [PATCH 712/752] make linter happy --- test/profile/test_nvtx.py | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/test/profile/test_nvtx.py b/test/profile/test_nvtx.py index ffe193e733cf..56e28a9c2e59 100644 --- a/test/profile/test_nvtx.py +++ b/test/profile/test_nvtx.py @@ -18,27 +18,27 @@ def test_nvtxit_base(torch_cuda_mock): @nvtxit() def call_b(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return 42 @nvtxit() def call_a(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return call_b() def dummy_func(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return call_a() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 dummy_func() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 # noqa: E501 assert torch_cuda_mock.nvtx.range_push.call_args_list == [ call('call_a_0'), call('call_b_0') ] @@ -52,27 +52,27 @@ def test_nvtxit_rename(torch_cuda_mock): @nvtxit() def call_b(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return 42 @nvtxit('a_nvtx') def call_a(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return call_b() def dummy_func(): - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 return call_a() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 dummy_func() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 # noqa: E501 assert torch_cuda_mock.nvtx.range_push.call_args_list == [ call('a_nvtx_0'), call('call_b_0') ] @@ -92,15 +92,15 @@ def call_b(): def call_a(): return call_b() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 call_b() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 # noqa: E501 call_a() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 2 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 2 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 2 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 2 # noqa: E501 assert torch_cuda_mock.nvtx.range_push.call_args_list == [ call('call_b_0'), call('call_a_0') @@ -121,15 +121,15 @@ def call_b(): def call_a(): return call_b() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 call_b() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 0 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 0 # noqa: E501 call_a() - assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 - assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStart.call_count == 1 # noqa: E501 + assert torch_cuda_mock.cudart.return_value.cudaProfilerStop.call_count == 1 # noqa: E501 assert torch_cuda_mock.nvtx.range_push.call_args_list == [ call('call_a_0'), call('call_b_1') From 137d7a26a012f33bf5c03f9f9213074062d0ee48 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 08:43:46 -0700 Subject: [PATCH 713/752] renamed some files to prevent misnomers --- docs/source/advanced/rag.rst | 2 +- .../{benchmark_model_archs.py => benchmark_model_archs_rag.py} | 0 examples/llm_plus_gnn/{ => nvtx_examples}/nvtx_run.sh | 0 .../llm_plus_gnn/{profiling_utils.py => rag_backend_utils.py} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename examples/llm_plus_gnn/{benchmark_model_archs.py => benchmark_model_archs_rag.py} (100%) rename examples/llm_plus_gnn/{ => nvtx_examples}/nvtx_run.sh (100%) rename examples/llm_plus_gnn/{profiling_utils.py => rag_backend_utils.py} (100%) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index 221c42f38932..d20623f87dab 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -416,7 +416,7 @@ from triplets. The code methods used in this tutorial can be found in .. code:: python from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet - from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader + from rag_construction_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader # We define this GraphStore to sample the neighbors of a node locally. # Ideally for a real remote backend, this interface would be replaced with an API to a Graph DB, such as Neo4j. diff --git a/examples/llm_plus_gnn/benchmark_model_archs.py b/examples/llm_plus_gnn/benchmark_model_archs_rag.py similarity index 100% rename from examples/llm_plus_gnn/benchmark_model_archs.py rename to examples/llm_plus_gnn/benchmark_model_archs_rag.py diff --git a/examples/llm_plus_gnn/nvtx_run.sh b/examples/llm_plus_gnn/nvtx_examples/nvtx_run.sh similarity index 100% rename from examples/llm_plus_gnn/nvtx_run.sh rename to examples/llm_plus_gnn/nvtx_examples/nvtx_run.sh diff --git a/examples/llm_plus_gnn/profiling_utils.py b/examples/llm_plus_gnn/rag_backend_utils.py similarity index 100% rename from examples/llm_plus_gnn/profiling_utils.py rename to examples/llm_plus_gnn/rag_backend_utils.py From fafbb9b196458e34a14550684ab1638e0194b361 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 10:47:20 -0700 Subject: [PATCH 714/752] typo --- examples/llm_plus_gnn/rag_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 14b116e5424f..420abee4c538 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -6,7 +6,7 @@ import pandas as pd import torch import tqdm -from profiling_utils import create_remote_backend_from_triplets +from rag_backend_utils import create_remote_backend_from_triplets from rag_feature_store import SentenceTransformerFeatureStore from rag_graph_store import NeighborSamplingRAGGraphStore From e14acc02b870301f6b21da58ddb416a45aed7952 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 11:44:51 -0700 Subject: [PATCH 715/752] typo 2 --- examples/llm_plus_gnn/rag_generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 420abee4c538..501f00f9fdc1 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -11,7 +11,7 @@ from rag_graph_store import NeighborSamplingRAGGraphStore from torch_geometric.data import Data -from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.datasets import WebQSPDataset from torch_geometric.datasets.web_qsp_dataset import ( preprocess_triplet, retrieval_via_pcst, @@ -28,7 +28,7 @@ args = parser.parse_args() # %% -ds = UpdatedWebQSPDataset("dataset", limit=args.num_samples) +ds = WebQSPDataset("dataset", limit=args.num_samples) # %% triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) From 62ce0b213a3083ebbfdcf0d8f4d14ade80a25fbe Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 12:47:35 -0700 Subject: [PATCH 716/752] typo 3 --- examples/llm_plus_gnn/benchmark_model_archs_rag.py | 4 ++-- examples/llm_plus_gnn/multihop/rag_generate_multihop.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs_rag.py b/examples/llm_plus_gnn/benchmark_model_archs_rag.py index 7737083b114a..ba30419552e1 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs_rag.py +++ b/examples/llm_plus_gnn/benchmark_model_archs_rag.py @@ -7,7 +7,7 @@ import torch from g_retriever import benchmark_models, get_loss, inference_step -from torch_geometric.datasets import UpdatedWebQSPDataset +from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever # %% @@ -36,7 +36,7 @@ # %% if not args.dataset_path: - ds = UpdatedWebQSPDataset('benchmark_archs') + ds = WebQSPDataset('benchmark_archs') else: # We just assume that the size of the dataset accomodates the # train/val/test split, because checking may be expensive. diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index eb06fb833166..ca881fb78877 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -17,7 +17,7 @@ sys.path.append('..') -from profiling_utils import create_remote_backend_from_triplets # noqa: E402 +from rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 from rag_feature_store import \ SentenceTransformerApproxFeatureStore # noqa: E402 from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 From a90b5d9d32277885ab9f12ebf8725365ba532a5b Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:34:03 -0700 Subject: [PATCH 717/752] fix retrieval test --- examples/llm_plus_gnn/rag_generate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 501f00f9fdc1..18b4a769694a 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -24,7 +24,7 @@ # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") parser.add_argument("--num_samples", type=int, default=4700) -parser.add_argument("--out_file", default="subg_results") +parser.add_argument("--out_file", default="subg_results.pt") args = parser.parse_args() # %% @@ -129,5 +129,5 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subg['question'] = questions[i] subg['label'] = ds[i]['label'] -pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file + '_metadata.csv') -torch.save(subgs, args.out_file + '.pt') +pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file.split('.')[0] + '_metadata.csv') +torch.save(subgs, args.out_file) From 018bdcff2bde8a8276280ac3033d11f6ba2de2ab Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:37:52 -0700 Subject: [PATCH 718/752] fix multihop --- examples/llm_plus_gnn/multihop/rag_generate_multihop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index ca881fb78877..6c53ca18484b 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -77,6 +77,7 @@ def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, # %% subgs = [] for subg in tqdm.tqdm(query_loader.query(q) for q in questions): + subg['question'] = q subgs.append(subg) torch.save(subgs, 'subg_results.pt') From 1ff34c840ef686af76830d7892f36bdb7e191b7e Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:40:08 -0700 Subject: [PATCH 719/752] remove ipynbs due to them being superseded by the rsts --- ...0_0_Encoding_a_Large_Knowledge_Graph.ipynb | 2055 ----------------- .../doc/0_1_Encoding_from_Scratch.ipynb | 836 ------- examples/llm_plus_gnn/doc/1_Retrieval.ipynb | 771 ------- examples/llm_plus_gnn/doc/_Introduction.ipynb | 128 - 4 files changed, 3790 deletions(-) delete mode 100644 examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb delete mode 100644 examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb delete mode 100644 examples/llm_plus_gnn/doc/1_Retrieval.ipynb delete mode 100644 examples/llm_plus_gnn/doc/_Introduction.ipynb diff --git a/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb b/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb deleted file mode 100644 index e1de6f2309c7..000000000000 --- a/examples/llm_plus_gnn/doc/0_0_Encoding_a_Large_Knowledge_Graph.ipynb +++ /dev/null @@ -1,2055 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Encoding a Large Knowledge Graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this notebook, we are going to walk through how to encode a large knowledge graph for the purposes of Graph RAG. We will provide two examples of how to do so, along with demonstration code." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 1: Building from Already Existing Datasets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In most RAG scenarios, the subset of the information corpus that gets retrieved is crucial for whether the appropriate response to the LLM. The same is true for GNN based RAG. Consider the following dataset WebQSP:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "\n", - "num_questions = 100\n", - "ds = UpdatedWebQSPDataset('small_sample', limit=num_questions)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "WebQSP is a dataset that is based off of a subset of the Freebase Knowledge Graph, which is an open-source knowledge graph formerly maintained by Google. For each question-answer pair in the dataset, a subgraph was chosen based on a Semantic SPARQL search on the larger knowledge graph, to provide relevent context on finding the answer. So each entry in the dataset consists of:\n", - "- A question to be answered\n", - "- The answer\n", - "- A knowledge graph subgraph of Freebase that has the context needed to answer the question." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Dataset({\n", - " features: ['id', 'question', 'answer', 'q_entity', 'a_entity', 'graph', 'choices'],\n", - " num_rows: 100\n", - "})" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.raw_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'WebQTrn-0',\n", - " 'question': 'what is the name of justin bieber brother',\n", - " 'answer': ['Jaxon Bieber'],\n", - " 'q_entity': ['Justin Bieber'],\n", - " 'a_entity': ['Jaxon Bieber'],\n", - " 'graph': [['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", - " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", - " ['Rudolph Valentino',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", - " ['Record producer',\n", - " 'music.performance_role.regular_performances',\n", - " 'm.012m1vf1'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell'],\n", - " ['2011 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0yrkr34'],\n", - " ['m.012bm2v1', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Stratford', 'location.location.containedby', 'Canada'],\n", - " ['Singer',\n", - " 'base.lightweight.profession.specialization_of',\n", - " 'Musicians and Singers'],\n", - " ['Enrique Iglesias', 'people.person.profession', 'Singer'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Akon', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['50 Cent', 'people.person.profession', 'Film Producer'],\n", - " ['As Long As You Love Me (Audien dubstep mix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kevin Risto', 'people.person.gender', 'Male'],\n", - " ['Classic Soul Network', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Shaggy', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['Mary J. Blige', 'people.person.profession', 'Record producer'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.12ml2glpn'],\n", - " ['Paul Anka', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0_w1gn3', 'award.award_nomination.nominated_for', 'Change Me'],\n", - " ['Baby', 'award.award_winning_work.awards_won', 'm.0n1ykxp'],\n", - " ['m.0njhxd_', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['1Club.FM: V101', 'broadcast.content.artist', 'The Roots'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0ghz3d6', 'tv.tv_guest_role.actor', 'Justin Bieber'],\n", - " ['American Music Award for Favorite Pop/Rock Album',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc259'],\n", - " ['A Michael Bublé Christmas', 'film.film.personal_appearances', 'm.0ng_vkd'],\n", - " ['Ontario', 'location.administrative_division.country', 'Canada'],\n", - " ['1Club.FM: Power', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['Music Producer', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Billboard Music Award for Top Streaming Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0njhx1b'],\n", - " ['Justin Bieber', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Heartbreaker', 'music.composition.recordings', 'Heartbreaker'],\n", - " ['Brandy Norwood', 'people.person.profession', 'Singer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0101ft2j'],\n", - " ['Justin Bieber', 'music.artist.album', 'All Bad'],\n", - " ['m.0n4rmg7', 'freebase.valuenotation.is_reviewed', 'Ceremony'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.episode',\n", - " 'Results Show: Week 7'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Britney Spears'],\n", - " ['One Less Lonely Girl',\n", - " 'music.album.primary_release',\n", - " 'One Less Lonely Girl'],\n", - " ['Twista', 'people.person.gender', 'Male'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Eminem'],\n", - " ['Ciara', 'broadcast.artist.content', 'FLOW 103'],\n", - " ['Jon M. Chu', 'film.director.film', \"Justin Bieber's Believe\"],\n", - " ['Leonardo DiCaprio', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0ndc3_1', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Somebody To Love', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Toby Gad', 'music.artist.genre', 'Rock music'],\n", - " ['Madonna', 'music.artist.genre', 'Pop music'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Europop'],\n", - " ['m.0gbm3cg',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Baby', 'music.recording.canonical_version', 'Baby'],\n", - " ['Contemporary R&B', 'music.genre.subgenre', 'Quiet Storm'],\n", - " ['Boyfriend', 'music.recording.artist', 'Justin Bieber'],\n", - " ['Dr. Dre', 'music.artist.genre', 'Rap music'],\n", - " ['MTV Video Music Award Japan for Best New Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhrwc'],\n", - " ['Beauty and a Beat', 'music.recording.featured_artists', 'Nicki Minaj'],\n", - " ['Hip hop music', 'broadcast.genre.content', 'FLOW 103'],\n", - " ['Maroon 5', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['m.0gctwjk',\n", - " 'tv.tv_guest_role.episodes_appeared_in',\n", - " 'Series 2, Episode 3'],\n", - " ['Enrique Iglesias', 'music.artist.genre', 'Dance-pop'],\n", - " ['Beauty and a Beast', 'music.recording.artist', 'Justin Bieber'],\n", - " ['FLOW 103', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Madonna', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Selena Gomez', 'people.person.profession', 'Dancer'],\n", - " ['Little Bird', 'music.recording.tracks', 'm.0v2hrym'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0t4s_bn'],\n", - " ['Never Say Never', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (PAULO & JACKINSKY radio)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Beauty and a Beat',\n", - " 'music.single.versions',\n", - " 'Beauty and a Beat (Wideboys Club Mix)'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Bryan Adams',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Madonna', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Gavin DeGraw', 'broadcast.artist.content', '1Club.FM: Mix 106'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0ndc259', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Terence Dudley', 'music.artist.genre', 'Reggae'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Actor'],\n", - " ['Adrienne Bailon', 'music.artist.genre', 'Pop music'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic music'],\n", - " ['Dany Brillant', 'people.person.gender', 'Male'],\n", - " ['Martin Kierszenbaum', 'people.person.gender', 'Male'],\n", - " ['Anastacia', 'people.person.nationality', 'United States of America'],\n", - " ['Amerie', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Gwen Stefani', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'music.composition.form', 'Song'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0yrkr34'],\n", - " ['Chef Tone', 'people.person.place_of_birth', 'Chicago'],\n", - " ['Dan Cutforth', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Whitney Houston'],\n", - " ['Record producer',\n", - " 'fictional_universe.character_occupation.characters_with_this_occupation',\n", - " 'Haley James Scott'],\n", - " ['Colbie Caillat', 'music.artist.genre', 'Pop music'],\n", - " ['C1', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pattie Mallette', 'people.person.spouse_s', 'm.0101gx29'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['Kanye West', 'people.person.profession', 'Singer'],\n", - " ['Pop music', 'common.topic.subject_of', 'Stephen Melton'],\n", - " ['radioIO Todays POP', 'broadcast.content.producer', 'Radioio'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Avril Lavigne', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['m.03vbp89', 'common.image.appears_in_topic_gallery', 'HitzRadio.com'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['Jennifer Lopez', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beauty And A Beat', 'music.composition.recordings', 'Beauty And A Beat'],\n", - " ['Rihanna', 'broadcast.artist.content', 'WildFMRadio.com'],\n", - " ['Adam Messinger', 'music.composer.compositions', 'Mistletoe'],\n", - " ['Live My Life', 'music.album.compositions', 'Live My Life'],\n", - " ['RedOne', 'music.artist.genre', 'Rock music'],\n", - " ['#thatPOWER', 'music.recording.canonical_version', '#thatPOWER'],\n", - " ['m.0yrjkl1', 'award.award_honor.honored_for', 'Baby'],\n", - " ['Terius Nash', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Little Bird', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['As Long As You Love Me (Ferry Corsten radio)',\n", - " 'music.recording.featured_artists',\n", - " 'Big Sean'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'HitzRadio.com'],\n", - " ['m.0gxnp5d', 'base.popstra.hangout.customer', 'Justin Bieber'],\n", - " ['Terius Nash', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'tv.tv_program_guest.appeared_on', 'm.0_grmr_'],\n", - " ['Athan Grace', 'people.person.profession', 'Actor'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Shorty Award for Music', 'award.award_category.nominees', 'm.0z3tqqt'],\n", - " ['All Around the World (acoustic version)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Bad Day', 'music.composition.composer', 'Marvin Isley'],\n", - " ['Brandy Norwood',\n", - " 'influence.influence_node.influenced_by',\n", - " 'Whitney Houston'],\n", - " ['Duffy', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['MTV Video Music Award for Artist to Watch',\n", - " 'award.award_category.winners',\n", - " 'm.0n1ykxp'],\n", - " ['Caitlin Beadles',\n", - " 'celebrities.celebrity.sexual_relationships',\n", - " 'm.0d33gyj'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audiobot instrumental)'],\n", - " ['Emphatic Radio.com!', 'common.topic.image', 'Emphatic Radio.com!'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg4h'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Scooter Braun'],\n", - " ['Lupe Fiasco', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Zac Efron', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'The Mighty Mighty Bosstones'],\n", - " ['m.012bm3j9', 'celebrities.friendship.friend', 'Rita Ora'],\n", - " ['Toby Gad', 'music.lyricist.lyrics_written', 'Beautiful'],\n", - " ['Lolly', 'music.composition.composer', 'Juicy J'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Documentary film'],\n", - " ['Timbaland', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0z1scxk', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Love Me', 'common.topic.notable_for', 'g.12h2xd7m9'],\n", - " ['Trey Songz', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Amerie', 'music.artist.genre', 'Pop music'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Beyoncé Knowles'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Y?N-Vee'],\n", - " ['Rodney Jerkins', 'music.artist.genre', 'Synthpop'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep edit)'],\n", - " ['Will Smith', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Recovery', 'music.recording.song', 'Recovery'],\n", - " ['Justin Timberlake', 'music.artist.genre', 'Electronic music'],\n", - " ['Mannie Fresh', 'people.person.nationality', 'United States of America'],\n", - " ['m.0101ftqp', 'film.film_crew_gig.film', \"Justin Bieber's Believe\"],\n", - " ['Benny Blanco', 'common.topic.notable_types', 'Record Producer'],\n", - " ['Leif Garrett', 'music.artist.genre', 'Rock music'],\n", - " ['Annette Funicello', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['First Dance', 'music.recording.artist', 'Justin Bieber'],\n", - " ['#thatPower', 'music.recording.song', '#thatPower'],\n", - " ['Children', 'rdf-schema#range', 'Person'],\n", - " ['Beautiful', 'common.topic.notable_for', 'g.1256glpl9'],\n", - " ['Kid Cudi', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Lady Gaga', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['2013 Teen Choice Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0wjgqck'],\n", - " ['The Island Def Jam Music Group',\n", - " 'organization.organization.parent',\n", - " 'm.04q65lb'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Rusted Root'],\n", - " ['radioIO RNB Mix', 'common.topic.notable_types', 'Broadcast Content'],\n", - " ['m.0z87d3n',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Red Carpet Fashion Icon Male'],\n", - " ['Shaffer Smith', 'music.artist.genre', 'Dance music'],\n", - " ['Live My Life', 'music.composition.composer', 'John Mamann'],\n", - " ['radioIO Classic RNB', 'broadcast.content.genre', 'Rock music'],\n", - " ['m.0njw4z2', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Ludacris', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Trick Daddy', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Yellowcard'],\n", - " ['Tampa', 'location.location.containedby', 'United States of America'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.compositions',\n", - " 'Love Never Felt So Good'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Nelly', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Marvin Isley', 'music.composer.compositions', 'Bad Day'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Composition'],\n", - " ['Katy Perry', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['Snoop Dogg', 'people.person.gender', 'Male'],\n", - " ['DMX', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0109_45q'],\n", - " ['Estelle', 'people.person.profession', 'Record producer'],\n", - " ['m.0_syttc', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['50 Cent', 'people.person.nationality', 'United States of America'],\n", - " ['Chris Jasper', 'people.person.gender', 'Male'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Pop music'],\n", - " ['Hot Wired Radio', 'broadcast.content.producer', 'Hot Wired Radio'],\n", - " ['m.0v_6zk4', 'tv.tv_guest_personal_appearance.person', 'Justin Bieber'],\n", - " ['Snoop Dogg',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['David Nicksay', 'people.person.gender', 'Male'],\n", - " ['Justin Bieber', 'people.person.profession', 'Record producer'],\n", - " ['Everlast', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Juno Awards of 2014',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0102z0vx'],\n", - " ['As Long As You Love Me (Audiobot remix)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['#thatPower', 'music.composition.composer', 'Will i Am'],\n", - " ['m.0gbm3bl', 'film.personal_film_appearance.person', 'Miley Cyrus'],\n", - " ['m.0_cyzs_',\n", - " 'celebrities.legal_entanglement.offense',\n", - " 'Driving under the influence'],\n", - " ['LeAnn Rimes', 'people.person.profession', 'Actor'],\n", - " ['KooL CrAzE', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Usher'],\n", - " ['Mann', 'people.person.gender', 'Male'],\n", - " ['JoJo', 'people.person.gender', 'Female'],\n", - " ['Right Here (featuring Drake)',\n", - " 'music.recording.canonical_version',\n", - " 'Right Here'],\n", - " ['Mason Levy', 'music.composer.compositions', 'Boyfriend'],\n", - " ['Beauty and a Beat', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0yrjynf',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Summer Music Star: Male'],\n", - " ['Pras', 'people.person.profession', 'Record producer'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Daniel Bedingfield'],\n", - " ['Hold Tight', 'award.award_nominated_work.award_nominations', 'm.0_w3zrs'],\n", - " ['My World 2.0', 'music.album.releases', 'My World 2.0'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Christmas in Washington', 'film.film.personal_appearances', 'm.0ng_k21'],\n", - " ['Marvin Isley',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Raekwon', 'broadcast.artist.content', 'Smoothbeats'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Adam Messinger', 'freebase.valuenotation.has_value', 'Date of birth'],\n", - " ['My World 2.0', 'common.topic.webpage', 'm.0cvc8k4'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctytd'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Toby Gad', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['School Gyrls', 'film.film.language', 'English Language'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boys / Boyfriend'],\n", - " ['Katy Perry', 'people.person.profession', 'Actor'],\n", - " ['As Long as You Love Me', 'common.topic.notable_for', 'g.125ddwtp0'],\n", - " ['Ronald Isley', 'people.person.profession', 'Actor'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.featured_artists',\n", - " 'Redfoo'],\n", - " ['HitzRadio.com', 'common.topic.webpage', 'm.03zb5cw'],\n", - " ['Jaxon Bieber', 'people.person.nationality', 'Canada'],\n", - " ['As Long as You Love Me (album version)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['Justin Bieber: Just Getting Started',\n", - " 'book.written_work.author',\n", - " 'Justin Bieber'],\n", - " ['BeirutNights.com Radio',\n", - " 'broadcast.content.artist',\n", - " 'Marc Maris vs. Ramone'],\n", - " ['Gwen Stefani', 'people.person.profession', 'Musician'],\n", - " ['m.0pcnqnb', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['m.0101fsyr', 'film.personal_film_appearance.person', 'Scooter Braun'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0102gvnb'],\n", - " ['Justin Bieber', 'music.featured_artist.recordings', '#Thatpower'],\n", - " ['Justin Bieber', 'celebrities.celebrity.net_worth', 'm.0yqflrk'],\n", - " ['Love Never Felt So Good',\n", - " 'music.album.releases',\n", - " 'Love Never Felt So Good'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['BeirutNights.com Radio', 'broadcast.content.artist', 'Soundlovers'],\n", - " ['Beauty and a Beat (DJ Laszlo Body Rock Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Sir Mix-a-Lot', 'people.person.profession', 'Actor'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Usher'],\n", - " ['Dance music',\n", - " 'broadcast.genre.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['1Club.FM: V101', 'broadcast.content.location', 'Chicago'],\n", - " ['Terius Nash', 'people.person.profession', 'Record producer'],\n", - " ['Terence Dudley', 'people.person.profession', 'Record producer'],\n", - " ['Mary J. Blige', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Baby', 'common.topic.notable_types', 'Award-Winning Work'],\n", - " ['Lolly', 'music.recording.canonical_version', 'Lolly'],\n", - " ['Scooter Braun', 'people.person.gender', 'Male'],\n", - " ['Mistletoe', 'music.album.artist', 'Justin Bieber'],\n", - " ['Sir Nolan', 'people.person.gender', 'Male'],\n", - " ['My Worlds: The Collection', 'music.album.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftt1'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Shiny Toy Guns'],\n", - " ['Synthpop', 'music.genre.parent_genre', 'K-pop'],\n", - " ['Adam Messinger',\n", - " 'music.composer.compositions',\n", - " \"Turn to You (Mother's Day Dedication)\"],\n", - " ['m.0yrktlv',\n", - " 'award.award_honor.award',\n", - " 'Teen Choice Award for Choice Male Hottie'],\n", - " ['Kanye West', 'people.person.nationality', 'United States of America'],\n", - " ['Iggy Azalea',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv4c'],\n", - " ['Juicy J', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['JellyRadio.com', 'broadcast.content.artist', 'DMX'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'The Black Eyed Peas'],\n", - " ['m.0gxnnzy',\n", - " 'celebrities.romantic_relationship.relationship_type',\n", - " 'Dated'],\n", - " ['Aaliyah', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Elvis Presley', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['radioIO Todays POP', 'common.topic.notable_for', 'g.1255g6pyx'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvcp'],\n", - " ['m.0njwb81', 'award.award_honor.award', 'UR Fave: New Artist'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Ashlee Simpson'],\n", - " ['L.A. Reid', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Britney Spears',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['m.0njhxd_', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Michael Jackson', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Frank Ocean', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vp800w'],\n", - " ['Singer', 'common.topic.subject_of', 'Justin Bieber'],\n", - " ['Fergie', 'music.artist.genre', 'Rock music'],\n", - " ['Gas Pedal', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Toby Gad', 'people.person.profession', 'Record producer'],\n", - " ['All Around The World', 'music.composition.composer', 'Justin Bieber'],\n", - " ['Mistletoe', 'music.album.release_type', 'Single'],\n", - " ['Kid Cudi', 'people.person.profession', 'Film Producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Ashley Tisdale'],\n", - " ['Somebody to Love (remix)', 'music.album.contributor', 'm.0vp7cl4'],\n", - " ['Live My Life (Party Rock remix)',\n", - " 'music.recording.tracks',\n", - " 'Live My Life (Party Rock remix)'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['m.0njw4z2',\n", - " 'award.award_honor.award',\n", - " 'MTV Europe Music Award for Best Male'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Snoop Dogg', 'people.person.profession', 'Record producer'],\n", - " ['Savan Kotecha', 'music.artist.genre', 'Dance-pop'],\n", - " ['m.0gbm3c3',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Rodney Jerkins', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Miley Cyrus'],\n", - " ['Pop music', 'base.schemastaging.music_genre_concept.artists', 'Yves Bole'],\n", - " [\"Destiny's Child\", 'music.artist.genre', 'Pop music'],\n", - " ['United States of America',\n", - " 'base.biblioness.bibs_topic.is_really',\n", - " 'United States of America'],\n", - " ['Christina Aguilera', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['m.09xx941', 'common.webpage.topic', 'Teen idol'],\n", - " ['Christina Milian', 'people.person.profession', 'Record producer'],\n", - " ['JoJo', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Next to You', 'music.album.release_type', 'Single'],\n", - " ['#thatPower', 'music.composition.recordings', '#thatPOWER'],\n", - " ['Willa Ford', 'people.person.languages', 'English Language'],\n", - " ['Frank Sinatra', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['All That Matters', 'music.composition.composer', 'Andre Harris'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', 'Smoothbeats'],\n", - " ['Paul Anka', 'music.artist.genre', 'Pop music'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Lady Gaga', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Jeremy Bieber', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Caitlin Beadles', 'people.person.nationality', 'Canada'],\n", - " ['m.0z8s_wn', 'award.award_honor.honored_for', 'My World'],\n", - " ['Favorite Girl', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Hot Wired Radio',\n", - " 'broadcast.content.broadcast',\n", - " 'Hot Wired Radio - 128kbps Stream'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'R. Kelly'],\n", - " ['Avery', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['m.0gbm3d9',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Ernie Isley', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Actor'],\n", - " ['m.0yrk18w', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Ja Rule', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Tupac Shakur', 'people.person.profession', 'Actor'],\n", - " ['Stephen Melton', 'common.topic.subjects', 'Singer-songwriter'],\n", - " ['Chris Brown', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Trick Daddy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Frank Ocean', 'people.person.nationality', 'United States of America'],\n", - " ['Christina Milian', 'music.composer.compositions', 'Baby'],\n", - " ['Chance the Rapper', 'music.artist.genre', 'Hip hop music'],\n", - " ['Justin Timberlake',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Khalil', 'people.person.gender', 'Male'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower (remix)'],\n", - " ['Recovery', 'freebase.valuenotation.is_reviewed', 'Initial release date'],\n", - " ['Selena Gomez',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Juelz Santana', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Fabolous', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Roller Coaster', 'common.topic.notable_for', 'g.1yp3bnqz7'],\n", - " ['m.0yrk4gn', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv7x'],\n", - " ['Jay Cassidy', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Anastacia', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['C1', 'music.artist.genre', 'Hip hop music'],\n", - " ['My Worlds Acoustic',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Album content type'],\n", - " ['m.0bvmhvb', 'common.webpage.resource', 'Justin Bieber Pictures'],\n", - " ['Live My Life', 'music.composition.language', 'English Language'],\n", - " ['Vocals', 'music.instrument.instrumentalists', 'Aaliyah'],\n", - " ['#thatPOWER', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Shorty Award for Celebrity', 'award.award_category.nominees', 'm.0y_g42w'],\n", - " ['Baby', 'music.album.releases', 'Baby'],\n", - " ['A Michael Bublé Christmas', 'common.topic.notable_types', 'Film'],\n", - " ['Right Here', 'music.recording.canonical_version', 'Right Here'],\n", - " ['Justin Bieber', 'people.person.profession', 'Musician'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Bigger', 'music.composition.composer', 'Waynne Nugent'],\n", - " ['Home to Mama', 'music.composition.composer', 'Cody Simpson'],\n", - " ['Big R Radio - The Hawk',\n", - " 'broadcast.content.artist',\n", - " 'The Black Eyed Peas'],\n", - " ['Thought Of You', 'music.composition.composer', 'Justin Bieber'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Singer', 'people.profession.specializations', 'Prima donna'],\n", - " ['Alanis Morissette', 'people.person.profession', 'Record producer'],\n", - " ['My World', 'award.award_nominated_work.award_nominations', 'm.0tkc3tj'],\n", - " ['Record producer', 'common.topic.notable_for', 'g.1258k9617'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0106bj25'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Mariah Carey', 'broadcast.artist.content', 'SoulfulSmoothJazz.com'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.production_companies',\n", - " 'AEG Live'],\n", - " ['Redfoo', 'people.person.gender', 'Male'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['WildFMRadio.com', 'broadcast.content.artist', '50 Cent'],\n", - " ['Ronald Isley', 'music.artist.genre', 'Quiet Storm'],\n", - " ['Nathan Lanier', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['P!nk', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Athan Grace', 'celebrities.celebrity.celebrity_friends', 'm.012r2w0k'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Right Here', 'music.album.featured_artists', 'Drake'],\n", - " ['m.01053qzf',\n", - " 'film.personal_film_appearance.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " ['Guglielmo Scilla', 'common.topic.notable_types', 'Person'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0v90skf'],\n", - " ['Jordan Pruitt', 'music.artist.genre', 'Pop music'],\n", - " ['Mason Levy', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Thought of You', 'common.topic.notable_types', 'Canonical Version'],\n", - " ['Whitney Houston', 'people.person.profession', 'Record producer'],\n", - " ['m.07lkzw7', 'common.webpage.category', 'Official Website'],\n", - " ['Ray J', 'people.person.profession', 'Musician'],\n", - " ['m.0gbmnvf', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Enrique Iglesias', 'people.person.gender', 'Male'],\n", - " ['m.0101fv5f',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Somebody to Love', 'music.composition.recordings', 'Somebody to Love'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Nelly'],\n", - " ['Eenie Meenie', 'music.single.versions', 'Eenie Meenie'],\n", - " ['Selena Gomez', 'music.artist.genre', 'Teen pop'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', 'Scooter Braun'],\n", - " ['Love Never Felt So Good', 'music.album.genre', 'Disco'],\n", - " ['Tupac Shakur', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.other_crew', 'm.0gbmntp'],\n", - " ['m.0p85jpp', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0v_729v',\n", - " 'tv.tv_guest_personal_appearance.appearance_type',\n", - " 'Guest host'],\n", - " ['Janet Jackson', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Christina Milian'],\n", - " ['Ja Rule', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Justin Bieber', 'music.featured_artist.albums', 'Runaway Love (remix)'],\n", - " ['RedOne', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['All Around the World', 'music.recording.featured_artists', 'Ludacris'],\n", - " ['Christina Milian', 'people.person.profession', 'Actor'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'The Pussycat Dolls'],\n", - " ['Dance music', 'broadcast.genre.content', '181-party'],\n", - " ['Queen Elizabeth II Diamond Jubilee Medal',\n", - " 'award.award_category.winners',\n", - " 'm.0njwqrb'],\n", - " ['Sean Kingston', 'people.person.profession', 'Singer'],\n", - " ['DMX', 'broadcast.artist.content', 'Hot 108 Jamz'],\n", - " ['Runaway Love (remix)', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['CMT Music Award: Collaborative Video of the Year',\n", - " 'award.award_category.winners',\n", - " 'm.0njvs9s'],\n", - " ['m.0yrkr34', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['One Time', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Soulja Boy'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Jupiter Rising'],\n", - " ['Katy Perry', 'music.artist.genre', 'Disco'],\n", - " ['Chingy', 'people.person.profession', 'Actor'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['The Notorious B.I.G.', 'music.artist.genre', 'Hip hop music'],\n", - " ['Dance music', 'broadcast.genre.content', 'Emphatic Radio.com!'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance-pop'],\n", - " ['Justin Bieber',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Contemporary R&B', 'common.topic.notable_types', 'Musical genre'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'City High'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0116rg0f'],\n", - " ['Chingy', 'people.person.gender', 'Male'],\n", - " ['Reed Smoot', 'people.person.gender', 'Male'],\n", - " [\"Justin Bieber's Believe\", 'film.film.edited_by', 'Jillian Twigger Moul'],\n", - " ['Teyana', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['All Bad', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.releases',\n", - " 'As Long As You Love Me (remixes)'],\n", - " ['Teen Choice Award for Choice Music: Breakout Artist - Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjvlh'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.010lkp2z'],\n", - " ['Singer', 'common.topic.article', 'm.09l6h'],\n", - " ['m.012r2w0k', 'celebrities.friendship.friend', 'Justin Bieber'],\n", - " ['Scooter Braun', 'film.producer.film', \"Justin Bieber's Believe\"],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'award.award_winning_work.awards_won',\n", - " 'm.0pc670l'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Beauty And A Beat', 'music.composition.form', 'Song'],\n", - " ['Britney Spears', 'music.artist.genre', 'Electronic dance music'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', \"Destiny's Child\"],\n", - " ['Beyoncé Knowles',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['m.0njhyh_',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Streaming Song (Video)'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Jeremy Bieber', 'people.person.children', 'Jazmyn Bieber'],\n", - " ['Ludacris', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_production_design_by',\n", - " 'Devorah Herbert'],\n", - " ['Bryan Adams', 'broadcast.artist.content', '1Club.FM: 80s (Pop)'],\n", - " ['m.0gbmntp', 'film.film_crew_gig.film', 'Justin Bieber: Never Say Never'],\n", - " ['Drake', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Pattie Mallette', 'base.popstra.organization.supporter', 'm.0gxnp72'],\n", - " ['Nick Jonas',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['justinbieber', 'award.award_winning_work.awards_won', 'm.0z0tmyv'],\n", - " ['Lupe Fiasco',\n", - " 'broadcast.artist.content',\n", - " \"PartyRadioUSA.net - Nation's #1 Party Authority\"],\n", - " ['Martin Kierszenbaum',\n", - " 'people.person.place_of_birth',\n", - " 'United States of America'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long as You Love Me'],\n", - " ['Juno Fan Choice Award', 'award.award_category.winners', 'm.0gwhmhm'],\n", - " ['m.0d_hbgr', 'common.webpage.category', 'Lyrics'],\n", - " ['Big Sean', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Beautiful', 'music.composition.lyricist', 'Toby Gad'],\n", - " ['Redfoo', 'music.artist.genre', 'Electronic dance music'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Snoop Dogg'],\n", - " ['K-Ci & JoJo', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Classic Soul Network', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['K-Ci & JoJo', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Stephen Melton', 'music.group_member.instruments_played', 'Vocals'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ['Twista', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '181-thebox'],\n", - " ['Jason Mraz', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Johntá Austin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['m.0y5tl39',\n", - " 'film.personal_film_appearance.film',\n", - " 'Les Coulisses des Golden Globes'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.09y89l2'],\n", - " ['m.0sgkyfg', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Kevin Risto', 'people.person.profession', 'Musician'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Kings of Leon'],\n", - " ['justinbieber',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0z0tgz6'],\n", - " ['Justin Bieber', 'music.artist.label', 'Island Records'],\n", - " ['Ernie Isley', 'people.person.nationality', 'United States of America'],\n", - " ['Kylie Minogue', 'people.person.profession', 'Film Producer'],\n", - " ['Yves Bole', 'tv.tv_actor.starring_roles', 'm.012bm2cn'],\n", - " ['Everlast', 'music.artist.label', 'Island Records'],\n", - " ['5th Annual Shorty Awards',\n", - " 'award.award_ceremony.awards_presented',\n", - " 'm.0ywvh8k'],\n", - " ['Chance the Rapper', 'music.featured_artist.albums', 'Confident'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Baby', 'common.topic.notable_types', 'Composition'],\n", - " ['Fabian', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Snoop Dogg', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['m.0tkqqgg',\n", - " 'award.award_nomination.award',\n", - " 'Juno Award for Pop Album of the Year'],\n", - " ['Ashlee Simpson', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Eenie Meenie', 'music.recording.canonical_version', 'Eenie Meenie'],\n", - " ['Person', 'type.type.properties', 'Parents'],\n", - " ['Bryan Adams', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Nasri', 'people.person.profession', 'Singer'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Vanessa Hudgens', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0njhx1b', 'award.award_honor.ceremony', '2011 Billboard Music Awards'],\n", - " ['As Long as You Love Me',\n", - " 'music.album.compositions',\n", - " 'As Long as You Love Me'],\n", - " ['Madonna', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['The Black Eyed Peas', 'music.artist.genre', 'Hip hop music'],\n", - " ['Bigger', 'music.composition.composer', 'Frank Ocean'],\n", - " ['Bigger', 'music.composition.recordings', 'Bigger'],\n", - " ['Canadian', 'common.topic.notable_types', 'Ethnicity'],\n", - " ['As Long as You Love Me', 'common.topic.article', 'm.0k0l2vk'],\n", - " ['Musician', 'freebase.equivalent_topic.equivalent_type', 'Musical Artist'],\n", - " ['Jennifer Lopez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'people.person.nationality', 'United States of America'],\n", - " ['Whitney Houston', 'music.artist.genre', 'Dance music'],\n", - " ['My Worlds Acoustic', 'music.album.album_content_type', 'Remix album'],\n", - " ['Avery', 'music.artist.label', 'The Island Def Jam Music Group'],\n", - " ['Change Me', 'music.album.primary_release', 'Change Me'],\n", - " ['Nick Jonas', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.film',\n", - " 'Zendaya: Behind the Scenes'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_4hw'],\n", - " ['That Should Be Me', 'music.composition.form', 'Song'],\n", - " ['Never Say Never', 'music.album.compositions', 'Never Say Never'],\n", - " ['m.09wsj7g', 'common.webpage.topic', 'Teen idol'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Justin Bieber'],\n", - " ['#thatPOWER', 'music.album.releases', '#thatPOWER'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Actor'],\n", - " ['Sir Nolan', 'music.artist.genre', 'Rock music'],\n", - " ['Beauty and a Beat (acoustic version)',\n", - " 'music.recording.song',\n", - " 'Beauty And A Beat'],\n", - " ['Ellen DeGeneres', 'people.person.nationality', 'United States of America'],\n", - " ['Sia Furler', 'people.person.profession', 'Singer-songwriter'],\n", - " ['Usher', 'music.composer.compositions', 'First Dance'],\n", - " ['m.0n1ykxp',\n", - " 'award.award_honor.award',\n", - " 'MTV Video Music Award for Artist to Watch'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'media_common.netflix_title.netflix_genres',\n", - " 'Rockumentary'],\n", - " ['Amerie', 'people.person.gender', 'Female'],\n", - " ['Real Change: Artists for Education',\n", - " 'film.film.personal_appearances',\n", - " 'm.0y5th3r'],\n", - " ['Mistletoe', 'music.album.primary_release', 'Mistletoe'],\n", - " ['Beautiful and the Beat',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['Baby', 'common.topic.notable_types', 'Musical Album'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Flyleaf'],\n", - " ['PYD', 'common.topic.notable_types', 'Composition'],\n", - " ['Ashlee Simpson', 'people.person.profession', 'Singer'],\n", - " ['Pray', 'music.album.artist', 'Justin Bieber'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0z8s562'],\n", - " ['Trey Songz', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Pras', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Coldplay'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Official website'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Model'],\n", - " ['iJustine', 'people.person.gender', 'Female'],\n", - " ['Nelly Furtado', 'people.person.gender', 'Female'],\n", - " ['Trey Songz', 'people.person.nationality', 'United States of America'],\n", - " ['m.0ng_vkd',\n", - " 'film.personal_film_appearance.film',\n", - " 'A Michael Bublé Christmas'],\n", - " [\"Justin Bieber's Believe\", 'film.film.produced_by', \"Bill O'Dowd\"],\n", - " ['m.0njhtjj', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Ludacris', 'music.composer.compositions', 'Baby'],\n", - " ['Terius Nash', 'music.featured_artist.recordings', 'Baby'],\n", - " ['Ginuwine', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Somebody to Love', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Vanessa Hudgens',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Mary J. Blige', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Record producer'],\n", - " ['#thatPOWER', 'music.recording.tracks', '#thatPower'],\n", - " ['m.0z8755b', 'award.award_honor.award_winner', 'Justin Bieber'],\n", - " ['Live My Life', 'common.topic.notable_for', 'g.1yl5pb70b'],\n", - " ['Contemporary R&B', 'broadcast.genre.content', '1Club.FM: V101'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'K-Ci & JoJo'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Shaggy', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Gas Pedal', 'music.recording.tracks', 'Gas Pedal'],\n", - " ['Jason Mraz', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Lady Antebellum'],\n", - " ['Ludacris', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Madonna', 'people.person.profession', 'Record producer'],\n", - " ['m.0yqfny6', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Keyshia Cole'],\n", - " ['1Club.FM: Power', 'broadcast.content.genre', 'Hip hop music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'M.I.A.'],\n", - " ['As Long as You Love Me (acoustic version)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Shaffer Smith', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Blu Cantrell', 'people.person.gender', 'Female'],\n", - " ['Contemporary R&B', 'common.topic.notable_for', 'g.125brs11z'],\n", - " ['Rob Thomas', 'people.person.gender', 'Male'],\n", - " ['Singer', 'people.profession.specializations', 'Piano Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.010b9gzv'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s_wn'],\n", - " ['m.0hvlt03',\n", - " 'film.film_film_distributor_relationship.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fvq6'],\n", - " ['Victoria Justice', 'base.popstra.celebrity.friendship', 'm.0cq9hwb'],\n", - " ['justinbieber',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0_srv2b'],\n", - " ['Terence Dudley', 'people.person.profession', 'Musician'],\n", - " ['Donna Summer',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['m.0101fszs',\n", - " 'film.personal_film_appearance.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['Alanis Morissette',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Official website'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lifehouse'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Jenna Andrews'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Cherish'],\n", - " ['Justin Timberlake', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Next to You', 'music.recording.song', 'Next to You'],\n", - " ['Victoria Justice', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Johnny Crawford', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Ray J', 'people.person.nationality', 'United States of America'],\n", - " ['Usher', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Madonna', 'influence.influence_node.influenced', 'Whitney Houston'],\n", - " ['m.0w3gbtv',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Montell Jordan', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nicki Minaj', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Fabolous', 'broadcast.artist.content', 'PowerHitz'],\n", - " ['Jessie J', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Jay-Z', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['Nelly Furtado',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Max Martin', 'freebase.valuenotation.has_value', 'Parents'],\n", - " ['Record producer', 'common.topic.webpage', 'm.09ygb05'],\n", - " ['As Long As You Love Me (Ferry Corsten remix)',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['m.0gbm3fj',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Bryan-Michael Cox',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Juvenile'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Ferry Corsten club dub)'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Synthpop'],\n", - " ['Tricky Stewart', 'common.topic.notable_types', 'Record Producer'],\n", - " ['As Long As You Love Me (Ferry Corsten club dub)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['#thatPOWER', 'music.album.album_content_type', 'Studio album'],\n", - " ['50 Cent', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Katy Perry', 'music.artist.genre', 'Electronic dance music'],\n", - " ['Kid Cudi', 'people.person.profession', 'Record producer'],\n", - " ['Hot Wired Radio', 'broadcast.content.artist', 'Miley Cyrus'],\n", - " ['m.0wfn4pm', 'people.sibling_relationship.sibling', 'Pattie Mallette'],\n", - " ['Kelly Clarkson', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Jaden Smith', 'people.person.profession', 'Dancer'],\n", - " ['m.0z8t2dy', 'award.award_nomination.nominated_for', 'My World'],\n", - " ['Keyshia Cole', 'people.person.profession', 'Record producer'],\n", - " ['Guest host',\n", - " 'tv.non_character_role.tv_guest_personal_appearances',\n", - " 'm.0v_98y7'],\n", - " ['Person', 'type.type.properties', 'Spouse (or domestic partner)'],\n", - " ['Fall Out Boy', 'music.artist.origin', 'Chicago'],\n", - " ['Jaxon Bieber', 'people.person.sibling_s', 'm.0gxnnwp'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'Hot 97.7'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Kelly Clarkson'],\n", - " ['FLOW 103', 'broadcast.content.artist', '50 Cent'],\n", - " ['Jordin Sparks', 'music.artist.genre', 'Dance-pop'],\n", - " ['L.A. Reid', 'music.producer.releases_produced', 'My World'],\n", - " ['L.A. Reid', 'people.person.gender', 'Male'],\n", - " ['Jessie J', 'music.artist.genre', 'Hip hop music'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'No Doubt'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Linkin Park'],\n", - " ['Beauty and a Beat (Bisbetic Radio Mix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['London', 'location.location.containedby', 'Ontario'],\n", - " ['Justin Bieber: Never Say Never',\n", - " 'film.film.film_set_decoration_by',\n", - " 'Lia Roldan'],\n", - " ['Bryan-Michael Cox', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Brown', 'music.composer.compositions', 'Next to You'],\n", - " ['Beautiful', 'music.recording.tracks', 'Beautiful'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gctwjk'],\n", - " ['Children', 'type.property.schema', 'Person'],\n", - " ['Change Me', 'music.album.releases', 'Change Me'],\n", - " ['RedOne', 'music.artist.label', 'Island Records'],\n", - " ['School Gyrls', 'film.film.starring', 'm.0jztshx'],\n", - " ['All Around the World',\n", - " 'music.recording.canonical_version',\n", - " 'All Around the World'],\n", - " ['m.0y5tl39', 'film.personal_film_appearance.person', 'Justin Bieber'],\n", - " ['Teen Choice Award for Choice Twitter Personality',\n", - " 'award.award_category.winners',\n", - " 'm.0wjhc6c'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['Live My Life', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['CL', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Miley Cyrus'],\n", - " ['m.0z8qqh5', 'award.award_nomination.award_nominee', 'Justin Bieber'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Will Smith'],\n", - " ['Emphatic Radio.com!', 'broadcast.content.artist', 'Baby Bash'],\n", - " ['Adrienne Bailon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"PartyRadioUSA.net - Nation's #1 Party Authority\",\n", - " 'broadcast.content.artist',\n", - " 'Lupe Fiasco'],\n", - " ['Hikaru Utada', 'music.artist.label', 'Island Records'],\n", - " ['Dr. Dre', 'people.person.profession', 'Record producer'],\n", - " ['Yves Bole', 'celebrities.celebrity.celebrity_friends', 'm.012bm4v7'],\n", - " ['Carrie Underwood', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Shaffer Smith', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Justin Bieber', 'music.composer.compositions', 'Change Me'],\n", - " ['Right Here', 'common.topic.notable_types', 'Composition'],\n", - " ['Change Me', 'music.composition.composer', 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Beauty and a Beat (Wideboys Radio Mix)',\n", - " 'music.recording.canonical_version',\n", - " 'Beauty and a Beat'],\n", - " ['Madonna', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#Thatpower', 'music.recording.artist', 'Will i Am'],\n", - " ['Award-Winning Work', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['m.0dm4cqr', 'celebrities.friendship.friend', 'Miley Cyrus'],\n", - " ['Scooter Braun', 'film.producer.film', 'Justin Bieber: Never Say Never'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0vp7qr5', 'music.recording_contribution.contributor', 'Jaden Smith'],\n", - " ['Eenie Meenie', 'music.recording.artist', 'Sean Kingston'],\n", - " ['m.0v90skf',\n", - " 'award.award_honor.award',\n", - " 'Billboard Music Award for Top Male Artist'],\n", - " ['Ludacris', 'people.person.profession', 'Actor'],\n", - " ['Heartbreaker', 'music.album.genre', 'Pop music'],\n", - " ['Cameo appearance',\n", - " 'tv.special_tv_performance_type.episode_performances',\n", - " 'm.0v1lwt2'],\n", - " ['Chef Tone', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Teen idol', 'common.topic.webpage', 'm.0b47zvy'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Sia Furler'],\n", - " ['Model', 'base.lightweight.profession.similar_professions', 'Actor'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Leona Lewis'],\n", - " ['Johntá Austin', 'music.lyricist.lyrics_written', 'Never Let You Go'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', 'Emphatic Radio.com!'],\n", - " ['m.0v_72tb', 'tv.tv_guest_personal_appearance.episode', 'Brown Family'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'One Chance'],\n", - " ['Never Let You Go', 'common.topic.notable_types', 'Composition'],\n", - " ['Live My Life', 'common.topic.article', 'm.0j4453y'],\n", - " ['Christina Milian', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " [\"Justin Bieber's Believe\", 'film.film.personal_appearances', 'm.0y5t8gm'],\n", - " ['Roller Coaster',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0_x4zg3'],\n", - " ['Chris Brown', 'people.person.nationality', 'United States of America'],\n", - " ['Justin Bieber: Never Say Never', 'film.film.produced_by', 'Jane Lipsitz'],\n", - " ['Lupe Fiasco', 'music.artist.genre', 'Hip hop music'],\n", - " ['Teen pop', 'common.topic.article', 'm.02ny8z'],\n", - " ['PowerHitz', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Iggy Azalea', 'people.person.gender', 'Female'],\n", - " ['Sia Furler', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Adrienne Bailon', 'people.person.profession', 'Dancer'],\n", - " ['Hip hop music', 'broadcast.genre.content', '181-beat'],\n", - " ['m.0sgk_cw',\n", - " 'award.award_honor.award',\n", - " \"Kids' Choice Award for Favorite Song\"],\n", - " ['Ray J', 'freebase.valuenotation.is_reviewed', 'Country of nationality'],\n", - " ['Beyoncé Knowles', 'broadcast.artist.content', 'Sunshine Radio'],\n", - " ['Iggy Azalea', 'music.artist.genre', 'Electronic dance music'],\n", - " ['MTV Video Music Brazil Award for Best International Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhhqv'],\n", - " ['Mariah Carey', 'music.artist.label', 'Island Records'],\n", - " ['Music', 'common.topic.subject_of', 'POPPMusic.net'],\n", - " ['Camagüey', 'common.topic.notable_types', 'City/Town/Village'],\n", - " ['Enrique Iglesias', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Favorite Girl', 'music.album.artist', 'Justin Bieber'],\n", - " ['m.0rqp4h0', 'music.track_contribution.track', 'Somebody to Love'],\n", - " ['Britney Spears', 'people.person.profession', 'Singer'],\n", - " ['Die in Your Arms', 'music.recording.song', 'Die in Your Arms'],\n", - " ['Britney Spears', 'freebase.valuenotation.is_reviewed', 'Children'],\n", - " ['Never Say Never', 'common.topic.notable_for', 'g.125bwly1y'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Rock music',\n", - " 'base.webvideo.internet_video_genre.series',\n", - " 'Biscuithands, The Animated Musical'],\n", - " ['Chris Brown', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chris Jasper', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Chef Tone', 'music.artist.genre', 'Hip hop music'],\n", - " ['Rudolph Isley', 'people.person.gender', 'Male'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Barry Weiss'],\n", - " ['Beauty and a Beat (Bisbetic Instrumental)',\n", - " 'common.topic.notable_types',\n", - " 'Musical Recording'],\n", - " ['MTV Europe Music Award for Best Male',\n", - " 'award.award_category.winners',\n", - " 'm.0z1scxk'],\n", - " ['Tricky Stewart', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Gwen Stefani'],\n", - " ['Will Smith', 'people.person.profession', 'Actor'],\n", - " ['Yves Bole', 'influence.influence_node.influenced_by', 'iJustine'],\n", - " ['Will i Am',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Boyfriend', 'music.composition.recordings', 'Boyfriend'],\n", - " ['Selena Gomez', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0y803nt', 'freebase.valuenotation.is_reviewed', 'Award winner'],\n", - " ['Fabian', 'people.person.gender', 'Male'],\n", - " ['SoulfulHipHop.com Radio', 'broadcast.content.artist', 'Mary J. Blige'],\n", - " ['Somebody to Love (remix)',\n", - " 'music.album.primary_release',\n", - " 'Somebody to Love (remix)'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Panic! at the Disco'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'Hot 108 Jamz'],\n", - " ['Eminem', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['#thatPOWER', 'music.single.versions', '#thatPOWER'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0102z0vx'],\n", - " ['Spouse', 'type.property.expected_type', 'Person'],\n", - " ['m.03zb5cw', 'common.webpage.topic', 'HitzRadio.com'],\n", - " ['Baby', 'music.recording.artist', 'Ludacris'],\n", - " ['Rudolph Valentino',\n", - " 'people.person.nationality',\n", - " 'United States of America'],\n", - " ['Hit-Boy', 'music.artist.genre', 'Hip hop music'],\n", - " ['Judy Garland',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Kelly Clarkson',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['#thatPower', 'music.composition.recordings', '#Thatpower'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'base.schemastaging.context_name.pronunciation',\n", - " 'm.011h9_22'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'Nelly'],\n", - " ['Miley Cyrus', 'people.person.profession', 'Musician'],\n", - " ['Justin Timberlake', 'people.person.gender', 'Male'],\n", - " ['#Thatpower', 'music.recording.tracks', '#thatPOWER'],\n", - " ['m.0vp8rhw',\n", - " 'music.recording_contribution.album',\n", - " 'Beauty and a Beat (Remixes)'],\n", - " ['Believe', 'award.award_nominated_work.award_nominations', 'm.0nhfd4m'],\n", - " ['Katy Perry: Part of Me',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['m.0jsmvv5',\n", - " 'film.film_regional_release_date.film',\n", - " 'Justin Bieber: Never Say Never'],\n", - " [\"Justin Bieber's Believe\", 'common.topic.notable_for', 'g.1yj4hbf4k'],\n", - " ['My Worlds: The Collection', 'music.album.release_type', 'Album'],\n", - " ['All Around The World (featuring Ludacris)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " ['Justin Bieber', 'base.popstra.celebrity.hangout', 'm.0gxnp5x'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Lady Gaga'],\n", - " ['1Club.FM: Mix 106', 'broadcast.content.producer', '1Club.FM'],\n", - " ['1Club.FM: Channel One', 'broadcast.content.artist', 'Duffy'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Dirty Vegas'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulClassics.com'],\n", - " ['Never Let You Go', 'music.composition.lyricist', 'Johntá Austin'],\n", - " ['m.0_x4zg3', 'award.award_nomination.nominated_for', 'Roller Coaster'],\n", - " ['Lady Antebellum', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['School Boy Records', 'music.record_label.artist', 'Madison Beer'],\n", - " [\"Justin Bieber's Believe\", 'film.film.other_crew', 'm.0101ftl5'],\n", - " ['Musical Album', 'freebase.type_hints.included_types', 'Topic'],\n", - " ['As Long As You Love Me',\n", - " 'music.single.versions',\n", - " 'As Long As You Love Me (Audien dubstep mix)'],\n", - " ['radioIO Todays RNB', 'broadcast.content.artist', 'Lil Wayne'],\n", - " ['Mary J. Blige', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Fergie', 'people.person.profession', 'Actor'],\n", - " ['Demi Lovato', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Stuart Ford', 'people.person.profession', 'Film Producer'],\n", - " ['Never Let You Go', 'music.composition.composer', 'Bryan-Michael Cox'],\n", - " ['Zac Efron', 'people.person.gender', 'Male'],\n", - " ['P!nk', 'music.artist.genre', 'Rock music'],\n", - " ['R. Kelly', 'people.person.profession', 'Film Producer'],\n", - " ['Gender', 'type.property.schema', 'Person'],\n", - " ['Adam Messinger', 'music.artist.genre', 'Rhythm and blues'],\n", - " ['Selena Gomez', 'influence.influence_node.influenced_by', 'Britney Spears'],\n", - " ['Right Here', 'common.topic.notable_for', 'g.12h31mb_7'],\n", - " ['JoJo', 'broadcast.artist.content', '1Club.FM: Channel One'],\n", - " ['Jessie J', 'influence.influence_node.influenced', 'Yves Bole'],\n", - " ['Under the Mistletoe',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Initial release date'],\n", - " ['Live My Life', 'music.recording.tracks', 'Live My Life'],\n", - " ['The Island Def Jam Music Group',\n", - " 'music.record_label.artist',\n", - " 'Slick Rick'],\n", - " ['Amerie', 'music.artist.genre', 'Rock music'],\n", - " ['Mary J. Blige', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.0pbzq13',\n", - " 'film.performance.special_performance_type',\n", - " 'Cameo appearance'],\n", - " ['Urban contemporary', 'broadcast.genre.content', 'SoulfulHipHop.com Radio'],\n", - " ['Height', 'type.property.unit', 'Meter'],\n", - " ['Iggy Azalea', 'people.person.profession', 'Model'],\n", - " ['NME Awards 2011', 'award.award_ceremony.awards_presented', 'm.0z8s562'],\n", - " ['Ray J',\n", - " 'freebase.valuenotation.has_no_value',\n", - " 'Spouse (or domestic partner)'],\n", - " ['Yves Bole', 'base.svocab.music_artist.genre', 'Pop'],\n", - " ['Mannie Fresh', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Boyfriend (acoustic version)',\n", - " 'music.recording.canonical_version',\n", - " 'Boyfriend'],\n", - " ['Big Sean', 'freebase.valuenotation.is_reviewed', 'Profession'],\n", - " ['Believe Tour',\n", - " 'music.concert_tour.album_or_release_supporting',\n", - " 'Believe'],\n", - " ['m.0yrk4gn', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Believe Acoustic', 'music.album.release_type', 'Album'],\n", - " ['Diplo', 'freebase.valuenotation.has_value', 'Height'],\n", - " ['Hikaru Utada', 'music.artist.genre', 'Synthpop'],\n", - " ['Roller Coaster', 'music.composition.composer', 'Julian Swirsky'],\n", - " ['Frank Ocean',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['As Long As You Love Me (Audiobot instrumental)',\n", - " 'music.recording.song',\n", - " 'As Long as You Love Me'],\n", - " ['Elvis Presley', 'music.artist.genre', 'Pop music'],\n", - " ['Lady Gaga', 'music.artist.genre', 'Pop music'],\n", - " ['FLOW 103', 'broadcast.content.artist', 'Shaffer Smith'],\n", - " ['Annette Funicello', 'base.icons.icon.icon_genre', 'Teen idol'],\n", - " ['Usher', 'people.person.nationality', 'United States of America'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Kelis', 'music.artist.genre', 'Contemporary R&B'],\n", - " [\"Justin Bieber's Believe\", 'film.film.release_date_s', 'm.0101fv5f'],\n", - " ['Don Henley', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Next to You', 'music.recording.tracks', 'Next to You'],\n", - " ['m.0gbm3b7',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Twista', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Sheryl Crow',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Gwen Stefani', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['All That Matters',\n", - " 'music.composition.composer',\n", - " 'Jason \\\\\"Poo Bear\\\\\" Boyd'],\n", - " ['Nasri', 'music.artist.genre', 'Reggae'],\n", - " ['#thatPOWER', 'music.recording.song', '#thatPower'],\n", - " ['Beauty and a Beat', 'common.topic.notable_types', 'Musical Album'],\n", - " ['m.0njdns_', 'award.award_honor.ceremony', 'American Music Awards of 2010'],\n", - " ['Yves Bole', 'base.schemastaging.music_artist_extra.genres', 'Europop'],\n", - " ['Bad 25', 'film.film.genre', 'Documentary film'],\n", - " ['Bigger', 'common.topic.image', '2009 Justin Bieber NYC 2'],\n", - " ['Jay-Z', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['As Long as You Love Me',\n", - " 'music.composition.recordings',\n", - " 'As Long As You Love Me'],\n", - " ['Fall Out Boy', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Musician'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'radioIO RNB Mix'],\n", - " ['Bryan-Michael Cox',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Whitney Houston', 'broadcast.artist.content', 'SoulfulHipHop.com Radio'],\n", - " ['Justin Bieber Videos', 'common.resource.annotations', 'm.0gc_9w6'],\n", - " ['Justin Bieber', 'tv.tv_actor.guest_roles', 'm.0gbcs1_'],\n", - " ['Chris Brown', 'broadcast.artist.content', 'radioIO Todays RNB'],\n", - " ['Coldplay', 'music.artist.genre', 'Rock music'],\n", - " ['Kevin Risto', 'people.person.profession', 'Record producer'],\n", - " ['Whitney Houston', 'people.person.profession', 'Model'],\n", - " ['Demi Lovato', 'freebase.valuenotation.has_no_value', 'Children'],\n", - " ['Coldplay', 'broadcast.artist.content', 'Big R Radio - Top 40 Hits'],\n", - " ['Anastacia', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['181-beat', 'broadcast.content.artist', 'Cassie Ventura'],\n", - " ['As Long as You Love Me',\n", - " 'music.recording.canonical_version',\n", - " 'As Long As You Love Me'],\n", - " ['Kylie Minogue', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Disney Parks Christmas Day Parade',\n", - " 'common.topic.notable_types',\n", - " 'Award-Winning Work'],\n", - " ['Ray J', 'people.person.profession', 'Artist'],\n", - " ['Avril Lavigne', 'people.person.profession', 'Singer-songwriter'],\n", - " ['American Music Award for Favorite Pop/Rock Male Artist',\n", - " 'award.award_category.winners',\n", - " 'm.0ndc0sf'],\n", - " ['Miley Cyrus', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['Music', 'common.topic.subject_of', 'Brian Keith Kennedy'],\n", - " ['The Notorious B.I.G.',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Place of birth'],\n", - " ['m.0njw1tn', 'freebase.valuenotation.is_reviewed', 'Year'],\n", - " ['Raekwon', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Christina Aguilera', 'music.artist.genre', 'Electronic music'],\n", - " ['PowerHitz', 'broadcast.content.artist', 'Outkast'],\n", - " ['U Smile', 'music.music_video.artist', 'Justin Bieber'],\n", - " ['HitzRadio.com', 'broadcast.content.genre', 'Rock music'],\n", - " ['Sean Kingston', 'music.artist.genre', 'Hip hop music'],\n", - " ['Nelly Furtado', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['Haley James Scott',\n", - " 'fictional_universe.fictional_character.occupation',\n", - " 'Record producer'],\n", - " ['Kylie Minogue', 'music.artist.genre', 'Rock music'],\n", - " ['Chris Jasper', 'people.person.nationality', 'United States of America'],\n", - " ['Ice Cube', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['My Worlds: The Collection',\n", - " 'music.album.album_content_type',\n", - " 'Compilation album'],\n", - " ['Lolly', 'music.album.releases', 'Lolly'],\n", - " ['Toby Gad', 'common.topic.notable_types', 'Record Producer'],\n", - " ['That Should Be Me', 'music.composition.lyricist', 'Adam Messinger'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Gavin DeGraw'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Sean Combs'],\n", - " ['m.0jvgmxc', 'freebase.valuenotation.has_no_value', 'Winning work'],\n", - " ['Christina Aguilera', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'DMX'],\n", - " ['Ja Rule', 'people.person.profession', 'Singer'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0111fg2d'],\n", - " ['Die in Your Arms',\n", - " 'award.award_nominated_work.award_nominations',\n", - " 'm.0z85qxq'],\n", - " ['Ashley Tisdale', 'people.person.profession', 'Singer-songwriter'],\n", - " ['m.012nv5gz', 'people.place_lived.location', 'Camagüey'],\n", - " ['Kuk Harrell',\n", - " 'film.person_or_entity_appearing_in_film.films',\n", - " 'm.0101ft5f'],\n", - " ['Somebody to Love (J Stax remix)',\n", - " 'music.recording.artist',\n", - " 'Justin Bieber'],\n", - " [\"Justin Bieber's Believe\",\n", - " 'film.film.executive_produced_by',\n", - " 'Allison Kaye Scarinzi'],\n", - " ['Adam Messinger', 'people.person.nationality', 'Canada'],\n", - " ['Nasri', 'music.artist.genre', 'Pop music'],\n", - " ['#thatPower', 'music.recording.featured_artists', 'Justin Bieber'],\n", - " ['The Island Def Jam Music Group', 'music.record_label.artist', 'Khalil'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Kid Cudi'],\n", - " ['C1', 'common.topic.notable_types', 'Musical Artist'],\n", - " ['.977 The Hits Channel', 'broadcast.content.artist', 'JoJo'],\n", - " ['School Boy Records', 'freebase.valuenotation.is_reviewed', 'Artists'],\n", - " ['Country', 'freebase.type_profile.strict_included_types', 'Topic'],\n", - " ['Kid Cudi', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Big Sean', 'music.featured_artist.recordings', 'As Long As You Love Me'],\n", - " ['Tricky Stewart', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['m.05sp405',\n", - " 'organization.organization_relationship.child',\n", - " 'Island Records'],\n", - " ['Savan Kotecha', 'people.person.profession', 'Record producer'],\n", - " ['Teen idol', 'base.icons.icon_genre.icons', 'Judy Garland'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Mariah Carey'],\n", - " ['m.0b47zvy', 'common.webpage.topic', 'Teen idol'],\n", - " ['John Mamann',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['Teen Choice Award for Choice Summer Music Star: Male',\n", - " 'award.award_category.winners',\n", - " 'm.0yrjynf'],\n", - " ['Juicy J', 'people.person.profession', 'Actor'],\n", - " ['m.0yqflrk',\n", - " 'measurement_unit.dated_money_value.source',\n", - " 'celebritynetworth.com'],\n", - " ['Miley Cyrus',\n", - " 'freebase.valuenotation.is_reviewed',\n", - " 'Country of nationality'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'Eminem'],\n", - " ['#thatPOWER', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['m.04q65lb',\n", - " 'organization.organization_relationship.child',\n", - " 'The Island Def Jam Music Group'],\n", - " ['Big Sean', 'people.person.nationality', 'United States of America'],\n", - " ['Beyoncé Knowles', 'people.person.profession', 'Film Producer'],\n", - " ['R. Kelly', 'broadcast.artist.content', '1Club.FM: V101'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', '\\\\\"Weird Al\\\\\" Yankovic'],\n", - " ['Geri Halliwell', 'people.person.profession', 'Actor'],\n", - " ['Aaliyah', 'broadcast.artist.content', 'Big R Radio - The Hawk'],\n", - " ['My World', 'music.album.artist', 'Justin Bieber'],\n", - " ['Don Henley', 'people.person.gender', 'Male'],\n", - " ['HitzRadio.com', 'broadcast.content.artist', 'Jay-Z'],\n", - " ['Musician', 'people.profession.specializations', 'Singer'],\n", - " ['Die in Your Arms',\n", - " 'music.recording.canonical_version',\n", - " 'Die in Your Arms'],\n", - " ['Chris Brown', 'broadcast.artist.content', '1Club.FM: Power'],\n", - " ['m.0njvs9s',\n", - " 'award.award_honor.award',\n", - " 'CMT Music Award: Collaborative Video of the Year'],\n", - " ['Dr. Dre', 'freebase.valuenotation.is_reviewed', 'Parents'],\n", - " ['Justin Bieber',\n", - " 'music.artist.album',\n", - " 'Turn to You (Mother’s Day Dedication)'],\n", - " ['Ludacris', 'music.artist.contribution', 'm.0vmyv4w'],\n", - " ['Bryan-Michael Cox', 'music.artist.genre', 'Contemporary R&B'],\n", - " ['City/Town/Village',\n", - " 'freebase.type_profile.strict_included_types',\n", - " 'Topic'],\n", - " ['Recovery', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Dancer', 'common.topic.notable_types', 'Profession'],\n", - " ['Live My Life', 'common.topic.notable_types', 'Musical Recording'],\n", - " ['Terence Dudley', 'people.person.gender', 'Male'],\n", - " ['Baby', 'music.composition.recordings', 'Polka Face'],\n", - " ['Lil Jon', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['BeirutNights.com Radio',\n", - " 'broadcast.content.artist',\n", - " 'Mr. Sosa & The Yayo'],\n", - " ['Whitney Houston', 'influence.influence_node.influenced_by', 'Yves Bole'],\n", - " ['Rihanna', 'music.artist.genre', 'Dance music'],\n", - " ['justinbieber', 'common.topic.notable_for', 'g.1yg57rnx6'],\n", - " ['SoulfulSmoothJazz.com', 'broadcast.content.genre', 'Contemporary R&B'],\n", - " ['Gender', 'type.property.expected_type', 'Gender'],\n", - " ['Geri Halliwell', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['m.0101fvbf',\n", - " 'film.film_regional_release_date.film',\n", - " \"Justin Bieber's Believe\"],\n", - " ['m.0yrhrwc',\n", - " 'award.award_honor.ceremony',\n", - " '2011 MTV Video Music Aid Japan'],\n", - " ['MTV Europe Music Award for Best North American Act',\n", - " 'award.award_category.winners',\n", - " 'm.0yrhmll'],\n", - " ['Iggy Azalea', 'freebase.valuenotation.is_reviewed', 'Height'],\n", - " ['m.0101ft1d',\n", - " 'film.personal_film_appearance.type_of_appearance',\n", - " 'Him/Herself'],\n", - " ['Big R Radio - Top 40 Hits', 'broadcast.content.artist', 'Fuel'],\n", - " ['Singer', 'base.descriptive_names.names.descriptive_name', 'm.0105_3yz'],\n", - " ['Diplo', 'freebase.valuenotation.is_reviewed', 'Date of birth'],\n", - " ['m.0f0dwc4', 'common.webpage.in_index', 'Blissful Master Index'],\n", - " ['Ciara', 'people.person.gender', 'Female'],\n", - " ['Big R Radio - The Hawk', 'broadcast.content.artist', 'Buckcherry'],\n", - " ['Britney Spears', 'music.artist.genre', 'Synthpop'],\n", - " ['Thought of You', 'music.recording.artist', 'Justin Bieber'],\n", - " ['m.0jzrrqs',\n", - " 'location.mailing_address.country',\n", - " 'United States of America'],\n", - " ['Justin Bieber', 'internet.blogger.blog', 'justinbieber'],\n", - " ['Live My Life', 'music.composition.recordings', 'Live My Life'],\n", - " ['Toby Gad', 'people.person.nationality', 'United States of America'],\n", - " ['Big R Radio - Top 40 Hits',\n", - " 'broadcast.content.artist',\n", - " 'Natasha Bedingfield'],\n", - " ['Hot Wired Radio', 'broadcast.content.genre', 'Rock music'],\n", - " ...],\n", - " 'choices': []}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.raw_dataset[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Although this dataset can be trained on as-is, a couple problems emerge from doing so:\n", - "1. A retrieval algorithm needs to be implemented and executed during inference time, that might not appropriately correspond to the algorithm that was used to generate the dataset subgraphs.\n", - "2. The dataset as is not stored computationally efficiently, as there will exist many duplicate nodes and edges that are shared between the questions." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As a result, it makes sense in this scenario to be able to encode all the entries into a large knowledge graph, so that duplicate nodes and edges can be avoided, and so that alternative retrieval algorithms can be tried. We can do this with the LargeGraphIndexer class:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from torch_geometric.data import LargeGraphIndexer, Data, get_features_for_triplets_groups\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "import time\n", - "import torch\n", - "import tqdm\n", - "from itertools import chain\n", - "import networkx as nx" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'), ('1Club.FM: Power', 'broadcast.content.artist', 'P!nk'), ('Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'), ('Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'), ('Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'), ('Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'), ('Stephen Melton', 'people.person.nationality', 'United States of America'), ('Record producer', 'music.performance_role.regular_performances', 'm.012m1vf1'), ('Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'), ('1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell')]\n" - ] - } - ], - "source": [ - "raw_dataset_graphs = [[tuple(trip) for trip in graph] for graph in ds.raw_dataset['graph']]\n", - "print(raw_dataset_graphs[0][:10])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To show the benefits of this indexer in action, we will use the following model to encode this sample of graphs using LargeGraphIndexer, along with naively." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cuda\n" - ] - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)\n", - "print(device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we compare the clock times of encoding using both methods." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [02:01<00:00, 1.22s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "121.68579435348511\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Indexing question-by-question\n", - "dataset_graphs_embedded = []\n", - "start = time.time()\n", - "for graph in tqdm.tqdm(raw_dataset_graphs):\n", - " nodes_map = dict()\n", - " edges_map = dict()\n", - " edge_idx_base = []\n", - "\n", - " for src, edge, dst in graph:\n", - " # Collect nodes\n", - " if src not in nodes_map:\n", - " nodes_map[src] = len(nodes_map)\n", - " if dst not in nodes_map:\n", - " nodes_map[dst] = len(nodes_map)\n", - " \n", - " # Collect edge types\n", - " if edge not in edges_map:\n", - " edges_map[edge] = len(edges_map)\n", - "\n", - " # Record edge\n", - " edge_idx_base.append((nodes_map[src], edges_map[edge], nodes_map[dst]))\n", - " \n", - " # Encode nodes and edges\n", - " sorted_nodes = list(sorted(nodes_map.keys(), key=lambda x: nodes_map[x]))\n", - " sorted_edges = list(sorted(edges_map.keys(), key=lambda x: edges_map[x]))\n", - "\n", - " x = model.encode(sorted_nodes, batch_size=256)\n", - " edge_attrs_map = model.encode(sorted_edges, batch_size=256)\n", - " \n", - " edge_attrs = []\n", - " edge_idx = []\n", - " for trip in edge_idx_base:\n", - " edge_attrs.append(edge_attrs_map[trip[1]])\n", - " edge_idx.append([trip[0], trip[2]])\n", - "\n", - " dataset_graphs_embedded.append(Data(x=x, edge_index=torch.tensor(edge_idx).T, edge_attr=torch.stack(edge_attrs, dim=0)))\n", - " \n", - " \n", - "print(time.time()-start)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Indexing...\n", - "Time to create whole knowledge_graph: 114.01080107688904\n", - "Retrieving Subgraphs...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [00:00<00:00, 212.87it/s]\n", - "100%|██████████| 100/100 [00:01<00:00, 80.90it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "114.66037964820862\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Using LargeGraphIndexer to make one large knowledge graph\n", - "from torch_geometric.data.large_graph_indexer import EDGE_RELATION\n", - "\n", - "start = time.time()\n", - "all_triplets_together = chain.from_iterable(raw_dataset_graphs)\n", - "# Index as one large graph\n", - "print('Indexing...')\n", - "indexer = LargeGraphIndexer.from_triplets(all_triplets_together)\n", - "\n", - "# first the nodes\n", - "unique_nodes = indexer.get_unique_node_features()\n", - "node_encs = model.encode(unique_nodes, batch_size=256)\n", - "indexer.add_node_feature(new_feature_name='x', new_feature_vals=node_encs)\n", - "\n", - "# then the edges\n", - "unique_edges = indexer.get_unique_edge_features(feature_name=EDGE_RELATION)\n", - "edge_attr = model.encode(unique_edges, batch_size=256)\n", - "indexer.add_edge_feature(new_feature_name=\"edge_attr\", new_feature_vals=edge_attr, map_from_feature=EDGE_RELATION)\n", - "\n", - "ckpt_time = time.time()\n", - "whole_knowledge_graph = indexer.to_data(node_feature_name='x', edge_feature_name='edge_attr') \n", - "whole_graph_done = time.time()\n", - "print(f\"Time to create whole knowledge_graph: {whole_graph_done-start}\")\n", - "\n", - "# Compute this to make sure we're comparing like to like on final time printout\n", - "whole_graph_diff = whole_graph_done-ckpt_time\n", - "\n", - "# retrieve subgraphs\n", - "print('Retrieving Subgraphs...')\n", - "dataset_graphs_embedded_largegraphindexer = [graph for graph in tqdm.tqdm(get_features_for_triplets_groups(indexer=indexer, triplet_groups=raw_dataset_graphs), total=num_questions)]\n", - "print(time.time()-start-whole_graph_diff)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The large graph indexer allows us to compute the entire knowledge graph from a series of samples, so that new retrieval methods can also be tested on the entire graph. We will see this attempted in practice later on." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's worth noting that, although the times are relatively similar right now, the speedup with largegraphindexer will be much higher as the size of the knowledge graph grows. This is due to the speedup being a factor of the number of unique nodes and edges in the graph." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024], pid=[100], e_pid=[100], node_idx=[1723], edge_idx=[9088]),\n", - " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024], pid=[100], e_pid=[100], node_idx=[1253], edge_idx=[4135]),\n", - " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024], pid=[100], e_pid=[100], node_idx=[1286], edge_idx=[2174]),\n", - " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024], pid=[100], e_pid=[100], node_idx=[1988], edge_idx=[5734]),\n", - " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024], pid=[100], e_pid=[100], node_idx=[633], edge_idx=[1490]),\n", - " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024], pid=[100], e_pid=[100], node_idx=[1047], edge_idx=[2772]),\n", - " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024], pid=[100], e_pid=[100], node_idx=[1383], edge_idx=[3987]),\n", - " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024], pid=[100], e_pid=[100], node_idx=[1064], edge_idx=[2456]),\n", - " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024], pid=[100], e_pid=[100], node_idx=[1030], edge_idx=[4162]),\n", - " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[6540]),\n", - " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024], pid=[100], e_pid=[100], node_idx=[1952], edge_idx=[5357]),\n", - " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024], pid=[100], e_pid=[100], node_idx=[1900], edge_idx=[5871]),\n", - " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024], pid=[100], e_pid=[100], node_idx=[1066], edge_idx=[3459]),\n", - " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024], pid=[100], e_pid=[100], node_idx=[1509], edge_idx=[4056]),\n", - " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024], pid=[100], e_pid=[100], node_idx=[2000], edge_idx=[4955]),\n", - " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024], pid=[100], e_pid=[100], node_idx=[1979], edge_idx=[4810]),\n", - " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024], pid=[100], e_pid=[100], node_idx=[1531], edge_idx=[5509]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", - " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024], pid=[100], e_pid=[100], node_idx=[574], edge_idx=[1664]),\n", - " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024], pid=[100], e_pid=[100], node_idx=[690], edge_idx=[2167]),\n", - " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024], pid=[100], e_pid=[100], node_idx=[1425], edge_idx=[3985]),\n", - " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024], pid=[100], e_pid=[100], node_idx=[851], edge_idx=[1934]),\n", - " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024], pid=[100], e_pid=[100], node_idx=[1618], edge_idx=[5270]),\n", - " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[7068]),\n", - " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024], pid=[100], e_pid=[100], node_idx=[1994], edge_idx=[4415]),\n", - " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[6744]),\n", - " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024], pid=[100], e_pid=[100], node_idx=[656], edge_idx=[1297]),\n", - " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024], pid=[100], e_pid=[100], node_idx=[881], edge_idx=[2168]),\n", - " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024], pid=[100], e_pid=[100], node_idx=[756], edge_idx=[1539]),\n", - " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024], pid=[100], e_pid=[100], node_idx=[1864], edge_idx=[8061]),\n", - " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024], pid=[100], e_pid=[100], node_idx=[1895], edge_idx=[5865]),\n", - " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024], pid=[100], e_pid=[100], node_idx=[873], edge_idx=[3519]),\n", - " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024], pid=[100], e_pid=[100], node_idx=[1816], edge_idx=[6375]),\n", - " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024], pid=[100], e_pid=[100], node_idx=[786], edge_idx=[1901]),\n", - " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024], pid=[100], e_pid=[100], node_idx=[885], edge_idx=[2366]),\n", - " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024], pid=[100], e_pid=[100], node_idx=[1228], edge_idx=[2634]),\n", - " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[3451]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", - " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024], pid=[100], e_pid=[100], node_idx=[977], edge_idx=[2903]),\n", - " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024], pid=[100], e_pid=[100], node_idx=[1401], edge_idx=[4570]),\n", - " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024], pid=[100], e_pid=[100], node_idx=[1168], edge_idx=[4004]),\n", - " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[8173]),\n", - " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024], pid=[100], e_pid=[100], node_idx=[1259], edge_idx=[4246]),\n", - " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024], pid=[100], e_pid=[100], node_idx=[1536], edge_idx=[8149]),\n", - " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024], pid=[100], e_pid=[100], node_idx=[1981], edge_idx=[6006]),\n", - " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024], pid=[100], e_pid=[100], node_idx=[1119], edge_idx=[4501]),\n", - " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024], pid=[100], e_pid=[100], node_idx=[1395], edge_idx=[7217]),\n", - " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024], pid=[100], e_pid=[100], node_idx=[983], edge_idx=[2642]),\n", - " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024], pid=[100], e_pid=[100], node_idx=[1634], edge_idx=[3905]),\n", - " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024], pid=[100], e_pid=[100], node_idx=[1182], edge_idx=[3135]),\n", - " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024], pid=[100], e_pid=[100], node_idx=[703], edge_idx=[1575]),\n", - " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024], pid=[100], e_pid=[100], node_idx=[194], edge_idx=[428]),\n", - " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024], pid=[100], e_pid=[100], node_idx=[876], edge_idx=[4971]),\n", - " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024], pid=[100], e_pid=[100], node_idx=[1964], edge_idx=[7721]),\n", - " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[5400]),\n", - " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024], pid=[100], e_pid=[100], node_idx=[1918], edge_idx=[6171]),\n", - " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024], pid=[100], e_pid=[100], node_idx=[1351], edge_idx=[3741]),\n", - " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024], pid=[100], e_pid=[100], node_idx=[475], edge_idx=[1488]),\n", - " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024], pid=[100], e_pid=[100], node_idx=[1990], edge_idx=[5011]),\n", - " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024], pid=[100], e_pid=[100], node_idx=[509], edge_idx=[986]),\n", - " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024], pid=[100], e_pid=[100], node_idx=[943], edge_idx=[2569]),\n", - " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024], pid=[100], e_pid=[100], node_idx=[739], edge_idx=[2404]),\n", - " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024], pid=[100], e_pid=[100], node_idx=[1674], edge_idx=[8595]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5444]),\n", - " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024], pid=[100], e_pid=[100], node_idx=[1223], edge_idx=[5361]),\n", - " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024], pid=[100], e_pid=[100], node_idx=[428], edge_idx=[1377]),\n", - " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024], pid=[100], e_pid=[100], node_idx=[1767], edge_idx=[4428]),\n", - " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024], pid=[100], e_pid=[100], node_idx=[404], edge_idx=[734]),\n", - " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024], pid=[100], e_pid=[100], node_idx=[1416], edge_idx=[4094]),\n", - " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024], pid=[100], e_pid=[100], node_idx=[1658], edge_idx=[6257]),\n", - " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024], pid=[100], e_pid=[100], node_idx=[1907], edge_idx=[7995]),\n", - " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024], pid=[100], e_pid=[100], node_idx=[1992], edge_idx=[4590]),\n", - " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024], pid=[100], e_pid=[100], node_idx=[645], edge_idx=[1666]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3280]),\n", - " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024], pid=[100], e_pid=[100], node_idx=[1956], edge_idx=[7203]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6926]),\n", - " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024], pid=[100], e_pid=[100], node_idx=[836], edge_idx=[1527]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024], pid=[100], e_pid=[100], node_idx=[1367], edge_idx=[3654]),\n", - " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024], pid=[100], e_pid=[100], node_idx=[1695], edge_idx=[5494]),\n", - " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024], pid=[100], e_pid=[100], node_idx=[371], edge_idx=[722]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024], pid=[100], e_pid=[100], node_idx=[1986], edge_idx=[6049]),\n", - " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024], pid=[100], e_pid=[100], node_idx=[815], edge_idx=[2322]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024], pid=[100], e_pid=[100], node_idx=[1026], edge_idx=[3285]),\n", - " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024], pid=[100], e_pid=[100], node_idx=[1233], edge_idx=[3088]),\n", - " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024], pid=[100], e_pid=[100], node_idx=[290], edge_idx=[577]),\n", - " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024], pid=[100], e_pid=[100], node_idx=[1358], edge_idx=[4891]),\n", - " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024], pid=[100], e_pid=[100], node_idx=[1946], edge_idx=[6642]),\n", - " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024], pid=[100], e_pid=[100], node_idx=[406], edge_idx=[1000]),\n", - " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024], pid=[100], e_pid=[100], node_idx=[1973], edge_idx=[5091]),\n", - " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024], pid=[100], e_pid=[100], node_idx=[1124], edge_idx=[4301]),\n", - " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024], pid=[100], e_pid=[100], node_idx=[1530], edge_idx=[4502]),\n", - " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024], pid=[100], e_pid=[100], node_idx=[1020], edge_idx=[2425]),\n", - " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024], pid=[100], e_pid=[100], node_idx=[1410], edge_idx=[8048]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024], pid=[100], e_pid=[100], node_idx=[1998], edge_idx=[5556]),\n", - " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024], pid=[100], e_pid=[100], node_idx=[1996], edge_idx=[4360]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024], pid=[100], e_pid=[100], node_idx=[1867], edge_idx=[4828]),\n", - " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024], pid=[100], e_pid=[100], node_idx=[1866], edge_idx=[5171]),\n", - " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024], pid=[100], e_pid=[100], node_idx=[293], edge_idx=[422])]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset_graphs_embedded_largegraphindexer" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Data(x=[1723, 1024], edge_index=[2, 9088], edge_attr=[9088, 1024]),\n", - " Data(x=[1253, 1024], edge_index=[2, 4135], edge_attr=[4135, 1024]),\n", - " Data(x=[1286, 1024], edge_index=[2, 2174], edge_attr=[2174, 1024]),\n", - " Data(x=[1988, 1024], edge_index=[2, 5734], edge_attr=[5734, 1024]),\n", - " Data(x=[633, 1024], edge_index=[2, 1490], edge_attr=[1490, 1024]),\n", - " Data(x=[1047, 1024], edge_index=[2, 2772], edge_attr=[2772, 1024]),\n", - " Data(x=[1383, 1024], edge_index=[2, 3987], edge_attr=[3987, 1024]),\n", - " Data(x=[1064, 1024], edge_index=[2, 2456], edge_attr=[2456, 1024]),\n", - " Data(x=[1030, 1024], edge_index=[2, 4162], edge_attr=[4162, 1024]),\n", - " Data(x=[1979, 1024], edge_index=[2, 6540], edge_attr=[6540, 1024]),\n", - " Data(x=[1952, 1024], edge_index=[2, 5357], edge_attr=[5357, 1024]),\n", - " Data(x=[1900, 1024], edge_index=[2, 5871], edge_attr=[5871, 1024]),\n", - " Data(x=[1066, 1024], edge_index=[2, 3459], edge_attr=[3459, 1024]),\n", - " Data(x=[1509, 1024], edge_index=[2, 4056], edge_attr=[4056, 1024]),\n", - " Data(x=[2000, 1024], edge_index=[2, 4955], edge_attr=[4955, 1024]),\n", - " Data(x=[1979, 1024], edge_index=[2, 4810], edge_attr=[4810, 1024]),\n", - " Data(x=[1531, 1024], edge_index=[2, 5509], edge_attr=[5509, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", - " Data(x=[574, 1024], edge_index=[2, 1664], edge_attr=[1664, 1024]),\n", - " Data(x=[690, 1024], edge_index=[2, 2167], edge_attr=[2167, 1024]),\n", - " Data(x=[1425, 1024], edge_index=[2, 3985], edge_attr=[3985, 1024]),\n", - " Data(x=[851, 1024], edge_index=[2, 1934], edge_attr=[1934, 1024]),\n", - " Data(x=[1618, 1024], edge_index=[2, 5270], edge_attr=[5270, 1024]),\n", - " Data(x=[1992, 1024], edge_index=[2, 7068], edge_attr=[7068, 1024]),\n", - " Data(x=[1994, 1024], edge_index=[2, 4415], edge_attr=[4415, 1024]),\n", - " Data(x=[1996, 1024], edge_index=[2, 6744], edge_attr=[6744, 1024]),\n", - " Data(x=[656, 1024], edge_index=[2, 1297], edge_attr=[1297, 1024]),\n", - " Data(x=[881, 1024], edge_index=[2, 2168], edge_attr=[2168, 1024]),\n", - " Data(x=[756, 1024], edge_index=[2, 1539], edge_attr=[1539, 1024]),\n", - " Data(x=[1864, 1024], edge_index=[2, 8061], edge_attr=[8061, 1024]),\n", - " Data(x=[1895, 1024], edge_index=[2, 5865], edge_attr=[5865, 1024]),\n", - " Data(x=[873, 1024], edge_index=[2, 3519], edge_attr=[3519, 1024]),\n", - " Data(x=[1816, 1024], edge_index=[2, 6375], edge_attr=[6375, 1024]),\n", - " Data(x=[786, 1024], edge_index=[2, 1901], edge_attr=[1901, 1024]),\n", - " Data(x=[885, 1024], edge_index=[2, 2366], edge_attr=[2366, 1024]),\n", - " Data(x=[1228, 1024], edge_index=[2, 2634], edge_attr=[2634, 1024]),\n", - " Data(x=[1358, 1024], edge_index=[2, 3451], edge_attr=[3451, 1024]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", - " Data(x=[977, 1024], edge_index=[2, 2903], edge_attr=[2903, 1024]),\n", - " Data(x=[1401, 1024], edge_index=[2, 4570], edge_attr=[4570, 1024]),\n", - " Data(x=[1168, 1024], edge_index=[2, 4004], edge_attr=[4004, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 8173], edge_attr=[8173, 1024]),\n", - " Data(x=[1259, 1024], edge_index=[2, 4246], edge_attr=[4246, 1024]),\n", - " Data(x=[1536, 1024], edge_index=[2, 8149], edge_attr=[8149, 1024]),\n", - " Data(x=[1981, 1024], edge_index=[2, 6006], edge_attr=[6006, 1024]),\n", - " Data(x=[1119, 1024], edge_index=[2, 4501], edge_attr=[4501, 1024]),\n", - " Data(x=[1395, 1024], edge_index=[2, 7217], edge_attr=[7217, 1024]),\n", - " Data(x=[983, 1024], edge_index=[2, 2642], edge_attr=[2642, 1024]),\n", - " Data(x=[1634, 1024], edge_index=[2, 3905], edge_attr=[3905, 1024]),\n", - " Data(x=[1182, 1024], edge_index=[2, 3135], edge_attr=[3135, 1024]),\n", - " Data(x=[703, 1024], edge_index=[2, 1575], edge_attr=[1575, 1024]),\n", - " Data(x=[194, 1024], edge_index=[2, 428], edge_attr=[428, 1024]),\n", - " Data(x=[876, 1024], edge_index=[2, 4971], edge_attr=[4971, 1024]),\n", - " Data(x=[1964, 1024], edge_index=[2, 7721], edge_attr=[7721, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 5400], edge_attr=[5400, 1024]),\n", - " Data(x=[1918, 1024], edge_index=[2, 6171], edge_attr=[6171, 1024]),\n", - " Data(x=[1351, 1024], edge_index=[2, 3741], edge_attr=[3741, 1024]),\n", - " Data(x=[475, 1024], edge_index=[2, 1488], edge_attr=[1488, 1024]),\n", - " Data(x=[1990, 1024], edge_index=[2, 5011], edge_attr=[5011, 1024]),\n", - " Data(x=[509, 1024], edge_index=[2, 986], edge_attr=[986, 1024]),\n", - " Data(x=[943, 1024], edge_index=[2, 2569], edge_attr=[2569, 1024]),\n", - " Data(x=[739, 1024], edge_index=[2, 2404], edge_attr=[2404, 1024]),\n", - " Data(x=[1674, 1024], edge_index=[2, 8595], edge_attr=[8595, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5444], edge_attr=[5444, 1024]),\n", - " Data(x=[1223, 1024], edge_index=[2, 5361], edge_attr=[5361, 1024]),\n", - " Data(x=[428, 1024], edge_index=[2, 1377], edge_attr=[1377, 1024]),\n", - " Data(x=[1767, 1024], edge_index=[2, 4428], edge_attr=[4428, 1024]),\n", - " Data(x=[404, 1024], edge_index=[2, 734], edge_attr=[734, 1024]),\n", - " Data(x=[1416, 1024], edge_index=[2, 4094], edge_attr=[4094, 1024]),\n", - " Data(x=[1658, 1024], edge_index=[2, 6257], edge_attr=[6257, 1024]),\n", - " Data(x=[1907, 1024], edge_index=[2, 7995], edge_attr=[7995, 1024]),\n", - " Data(x=[1992, 1024], edge_index=[2, 4590], edge_attr=[4590, 1024]),\n", - " Data(x=[645, 1024], edge_index=[2, 1666], edge_attr=[1666, 1024]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3280], edge_attr=[3280, 1024]),\n", - " Data(x=[1956, 1024], edge_index=[2, 7203], edge_attr=[7203, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6926], edge_attr=[6926, 1024]),\n", - " Data(x=[836, 1024], edge_index=[2, 1527], edge_attr=[1527, 1024]),\n", - " Data(x=[1367, 1024], edge_index=[2, 3654], edge_attr=[3654, 1024]),\n", - " Data(x=[1695, 1024], edge_index=[2, 5494], edge_attr=[5494, 1024]),\n", - " Data(x=[371, 1024], edge_index=[2, 722], edge_attr=[722, 1024]),\n", - " Data(x=[1986, 1024], edge_index=[2, 6049], edge_attr=[6049, 1024]),\n", - " Data(x=[815, 1024], edge_index=[2, 2322], edge_attr=[2322, 1024]),\n", - " Data(x=[1026, 1024], edge_index=[2, 3285], edge_attr=[3285, 1024]),\n", - " Data(x=[1233, 1024], edge_index=[2, 3088], edge_attr=[3088, 1024]),\n", - " Data(x=[290, 1024], edge_index=[2, 577], edge_attr=[577, 1024]),\n", - " Data(x=[1358, 1024], edge_index=[2, 4891], edge_attr=[4891, 1024]),\n", - " Data(x=[1946, 1024], edge_index=[2, 6642], edge_attr=[6642, 1024]),\n", - " Data(x=[406, 1024], edge_index=[2, 1000], edge_attr=[1000, 1024]),\n", - " Data(x=[1973, 1024], edge_index=[2, 5091], edge_attr=[5091, 1024]),\n", - " Data(x=[1124, 1024], edge_index=[2, 4301], edge_attr=[4301, 1024]),\n", - " Data(x=[1530, 1024], edge_index=[2, 4502], edge_attr=[4502, 1024]),\n", - " Data(x=[1020, 1024], edge_index=[2, 2425], edge_attr=[2425, 1024]),\n", - " Data(x=[1410, 1024], edge_index=[2, 8048], edge_attr=[8048, 1024]),\n", - " Data(x=[1998, 1024], edge_index=[2, 5556], edge_attr=[5556, 1024]),\n", - " Data(x=[1996, 1024], edge_index=[2, 4360], edge_attr=[4360, 1024]),\n", - " Data(x=[1867, 1024], edge_index=[2, 4828], edge_attr=[4828, 1024]),\n", - " Data(x=[1866, 1024], edge_index=[2, 5171], edge_attr=[5171, 1024]),\n", - " Data(x=[293, 1024], edge_index=[2, 422], edge_attr=[422, 1024])]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset_graphs_embedded" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We expect the two results to be functionally identical, with the differences being due to floating point jitter." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def results_are_close_enough(ground_truth: Data, new_method: Data, thresh=.8):\n", - " def _sorted_tensors_are_close(tensor1, tensor2):\n", - " return torch.all(torch.isclose(tensor1.sort(dim=0)[0], tensor2.sort(dim=0)[0]).float().mean(axis=1) > thresh)\n", - " def _graphs_are_same(tensor1, tensor2):\n", - " return nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor1.T)) == nx.weisfeiler_lehman_graph_hash(nx.Graph(tensor2.T))\n", - " return _sorted_tensors_are_close(ground_truth.x, new_method.x) \\\n", - " and _sorted_tensors_are_close(ground_truth.edge_attr, new_method.edge_attr) \\\n", - " and _graphs_are_same(ground_truth.edge_index, new_method.edge_index)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [00:25<00:00, 4.00it/s]\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_results_match = True\n", - "for old_graph, new_graph in tqdm.tqdm(zip(dataset_graphs_embedded, dataset_graphs_embedded_largegraphindexer), total=num_questions):\n", - " all_results_match &= results_are_close_enough(old_graph, new_graph)\n", - "all_results_match" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When scaled up to the entire dataset, we see a 2x speedup with indexing this way.\n", - "\n", - "WebQSPDataset is a question-by-question implementation.\n", - "\n", - "UpdatedQSPDataset is a LargeGraphIndexer implementation.\n", - "\n", - "These were computed on an RTX 4090 with 24GB of memory. Your milage may vary." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example, we will take a set of multi-hop questions, and combine them with an existing Wikidata knowledge graph to produce a new dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To be continued in 0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb b/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb deleted file mode 100644 index 0188a025ba92..000000000000 --- a/examples/llm_plus_gnn/doc/0_1_Encoding_from_Scratch.ipynb +++ /dev/null @@ -1,836 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Encoding A Large Knowledge Graph Part 2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this notebook, we will continue where we left off by building a new multi-hop QA dataset based on Wikidata." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 2: Building a new Dataset from Questions and an already-existing Knowledge Graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Motivation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One potential application of knowledge graph structural encodings is capturing the relationships between different entities that are multiple hops apart. This can be challenging for an LLM to recognize from prepended graph information. Here's a motivating example (credit to @Rishi Puri):" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import SVG" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SVG(filename='./media/multihop_example.svg')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example, the question can only be answered by reasoning about the relationships between the entities in the knowledge graph." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Building a Multi-Hop QA Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To start, we need to download the raw data of a knowledge graph. In this case, we use WikiData5M ([Wang et al](https://paperswithcode.com/paper/kepler-a-unified-model-for-knowledge)). Here we download the raw triplets and their entity codes. Information about this dataset can be found [here](https://deepgraphlearning.github.io/project/wikidata5m)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following download contains the ID to plaintext mapping for all the entities and relations in the knowledge graph:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget -O \"https://www.dropbox.com/s/lnbhc8yuhit4wm5/wikidata5m_alias.tar.gz\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!tar -xvf \"wikidata5m_alias.tar.gz\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with open('wikidata5m_entity.txt') as f:\n", - " print(f.readline())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with open('wikidata5m_relation.txt') as f:\n", - " print(f.readline())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And then this download contains the raw triplets:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget -O \"https://www.dropbox.com/s/563omb11cxaqr83/wikidata5m_all_triplet.txt.gz\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!gzip -d \"wikidata5m_all_triplet.txt.gz\" -f" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with open('wikidata5m_all_triplet.txt') as f:\n", - " print(f.readline())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To start, we are going to preprocess the knowledge graph to substitute each of the entity/relation codes with their plaintext aliases. This makes it easier to use a pre-trained textual encoding model to create triplet embeddings, as such a model likely won't understand how to properly embed the entity codes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import tqdm\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Substitute entity codes with their aliases\n", - "# Picking the first alias for each entity (rather arbitrarily)\n", - "alias_map = {}\n", - "rel_alias_map = {}\n", - "for line in open('wikidata5m_entity.txt'):\n", - " parts = line.strip().split('\\t')\n", - " entity_id = parts[0]\n", - " aliases = parts[1:]\n", - " alias_map[entity_id] = aliases[0]\n", - "for line in open('wikidata5m_relation.txt'):\n", - " parts = line.strip().split('\\t')\n", - " relation_id = parts[0]\n", - " relation_name = parts[1]\n", - " rel_alias_map[relation_id] = relation_name" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "full_graph = []\n", - "missing_total = 0\n", - "total = 0\n", - "for line in tqdm.tqdm(open('wikidata5m_all_triplet.txt')):\n", - " src, rel, dst = line.strip().split('\\t')\n", - " if src not in alias_map:\n", - " missing_total += 1\n", - " if dst not in alias_map:\n", - " missing_total += 1\n", - " if rel not in rel_alias_map:\n", - " missing_total += 1\n", - " total += 3\n", - " full_graph.append([alias_map.get(src, src), rel_alias_map.get(rel, rel), alias_map.get(dst, dst)])\n", - "print(f\"Missing aliases: {missing_total}/{total}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "full_graph[:10]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now `full_graph` represents the knowledge graph triplets in understandable plaintext." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we need a set of multi-hop questions that the Knowledge Graph will provide us with context for. We utilize a subset of [HotPotQA](https://hotpotqa.github.io/) ([Yang et. al.](https://arxiv.org/pdf/1809.09600)) called [2WikiMultiHopQA](https://github.com/Alab-NII/2wikimultihop) ([Ho et. al.](https://aclanthology.org/2020.coling-main.580.pdf)), which includes a subgraph of entities that serve as the ground truth justification for answering each multi-hop question:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget -O \"https://www.dropbox.com/s/ms2m13252h6xubs/data_ids_april7.zip\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!unzip -o \"data_ids_april7.zip\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with open('train.json') as f:\n", - " train_data = json.load(f)\n", - "train_df = pd.DataFrame(train_data)\n", - "train_df['split_type'] = 'train'\n", - "\n", - "with open('dev.json') as f:\n", - " dev_data = json.load(f)\n", - "dev_df = pd.DataFrame(dev_data)\n", - "dev_df['split_type'] = 'dev'\n", - "\n", - "with open('test.json') as f:\n", - " test_data = json.load(f)\n", - "test_df = pd.DataFrame(test_data)\n", - "test_df['split_type'] = 'test'\n", - "\n", - "df = pd.concat([train_df, dev_df, test_df])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['split_type'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['type'].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we need to extract the subgraphs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['graph_size'] = df['evidences_id'].apply(lambda row: len(row))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['graph_size'].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "(Optional) We take only questions where the evidence graph is greater than 0. (Note: this gets rid of the test set):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# df = df[df['graph_size'] > 0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['split_type'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df.columns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df = df[['_id', 'question', 'answer', 'split_type', 'evidences_id', 'type', 'graph_size']]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Checkpoint:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df.to_csv('wikimultihopqa_refined.csv', index=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we need to check that all the entities mentioned in the question/answer set are also present in the Wikidata graph:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "relation_map = {}\n", - "with open('wikidata5m_relation.txt') as f:\n", - " for line in tqdm.tqdm(f):\n", - " parts = line.strip().split('\\t')\n", - " for i in range(1, len(parts)):\n", - " if parts[i] not in relation_map:\n", - " relation_map[parts[i]] = []\n", - " relation_map[parts[i]].append(parts[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Manually check to see if all of these are valid in WikiData DB, even if they may not be answerable in WikiData5M\n", - "for row in refined_df.itertuples():\n", - " for trip in row.evidences_id:\n", - " relation = trip[1]\n", - " if relation not in relation_map:\n", - " print(f'The following relation is not found: {relation}')\n", - " elif len(relation_map[relation]) > 1:\n", - " print(f'The following relation alias has a collision: {relation}: {relation_map[relation]}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "entity_set = set()\n", - "with open('wikidata5m_entity.txt') as f:\n", - " for line in tqdm.tqdm(f):\n", - " entity_set.add(line.strip().split('\\t')[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "missing_entities = set()\n", - "missing_entity_idx = set()\n", - "for i, row in enumerate(refined_df.itertuples()):\n", - " for trip in row.evidences_id:\n", - " if len(trip) != 3:\n", - " print(trip)\n", - " entities = trip[0], trip[2]\n", - " for entity in entities:\n", - " if entity not in entity_set:\n", - " print(f'The following entity was not found in the KG: {entity}')\n", - " missing_entities.add(entity)\n", - " missing_entity_idx.add(i)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Right now, we drop the missing entity entries. Additional preprocessing can be done here to resolve the entity/relation collisions, but that is out of the scope for this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "len(missing_entity_idx)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "refined_df.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# missing relations are ok, but missing entities cannot be mapped to plaintext, so they should be dropped.\n", - "refined_df.reset_index(inplace=True, drop=True)\n", - "refined_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cleaned_df = refined_df.drop(missing_entity_idx)\n", - "cleaned_df['split_type'].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we save the resulting graph and questions/answers dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cleaned_df.to_csv('wikimultihopqa_cleaned.csv', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "torch.save(full_graph, 'wikimultihopqa_full_graph.pt')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Question: How do we extract a contextual subgraph for a given query?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The chosen retrieval algorithm is a critical component in the pipeline for affecting RAG performance. In the next section (1), we will demonstrate a naive method of retrieval for a large knowledge graph, and how to apply it to this dataset along with WebQSP." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Preparing a Textualized Graph for LLM" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For now however, we need to prepare the graph data to be used as a plaintext prefix to the LLM. In order to do this, we want to prompt the LLM to use the unique nodes, and unique edge triplets of a given subgraph. In order to do this, we prepare a unique indexed node df and edge df for the knowledge graph now. This process occurs trivially with the LargeGraphIndexer:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from torch_geometric.data import LargeGraphIndexer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "indexer = LargeGraphIndexer.from_triplets(full_graph)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Node DF\n", - "textual_nodes = pd.DataFrame.from_dict(\n", - " {\"node_attr\": indexer.get_node_features()})\n", - "textual_nodes[\"node_id\"] = textual_nodes.index\n", - "textual_nodes = textual_nodes[[\"node_id\", \"node_attr\"]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "textual_nodes.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notice how LargeGraphIndexer ensures that there are no duplicate indices:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "textual_nodes['node_attr'].unique().shape[0]/textual_nodes.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Edge DF\n", - "textual_edges = pd.DataFrame(indexer.get_edge_features(),\n", - " columns=[\"src\", \"edge_attr\", \"dst\"])\n", - "textual_edges[\"src\"] = [\n", - " indexer._nodes[h] for h in textual_edges[\"src\"]\n", - "]\n", - "textual_edges[\"dst\"] = [\n", - " indexer._nodes[h] for h in textual_edges[\"dst\"]\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: The edge table refers to each node by its index in the node table. We will see how this gets utilized later when indexing a subgraph." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "textual_edges.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can save the result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "textual_nodes.to_csv('wikimultihopqa_textual_nodes.csv', index=False)\n", - "textual_edges.to_csv('wikimultihopqa_textual_edges.csv', index=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now were done! This knowledge graph and dataset will get used later on in Section 1." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: Refactor everything below this point into its own notebook" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Generating Subgraphs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from profiling_utils import create_remote_backend_from_triplets\n", - "from rag_feature_store import SentenceTransformerFeatureStore\n", - "from rag_graph_store import NeighborSamplingRAGGraphStore\n", - "from torch_geometric.loader import RAGQueryLoader\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.updated_web_qsp_dataset import preprocess_triplet, retrieval_via_pcst\n", - "from torch_geometric.data import get_features_for_triplets_groups, Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(model_name='sentence-transformers/all-roberta-large-v1').to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fs, gs = create_remote_backend_from_triplets(full_graph, model, model, NeighborSamplingRAGGraphStore, SentenceTransformerFeatureStore, 'encode', 'encode', preprocess_triplet, 'wikidata_graph', node_method_kwargs={\"batch_size\": 256}).load()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "graphs = torch.load('subg_results.pt')\n", - "graph_df = pd.read_csv(\"wikimultihopqa_cleaned.csv\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "graph_df['is_train'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "graph_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "graph_df['is_train'].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb b/examples/llm_plus_gnn/doc/1_Retrieval.ipynb deleted file mode 100644 index 99fc723921e3..000000000000 --- a/examples/llm_plus_gnn/doc/1_Retrieval.ipynb +++ /dev/null @@ -1,771 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Retrieval Algorithms and Scaling Retrieval" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Motivation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When building a RAG Pipeline for inference, the retrieval component is important for the following reasons:\n", - "1. A given algorithm for retrieving subgraph context can have a marked effect on the hallucination rate of the responses in the model\n", - "2. A given retrieval algorithm needs to be able to scale to larger graphs of millions of nodes and edges in order to be practical for production." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this notebook, we will explore how to construct a RAG retrieval algorithm from a given subgraph, and conduct some experiments to evaluate its runtime performance." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We want to do so in-line with Pytorch Geometric's in-house framework for remote backends:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABJkAAAEdCAYAAACv07ZCAAABYmlDQ1BJQ0MgUHJvZmlsZQAAKJFtkMFLAlEQxj/LEHSJhIgOBXsLw8LWQLoEZlGBh8WKstu6rmul9tjdCG8dgs5B1H8Q/QGBBB46d4ggSCiKDuE5CPZSss3TarWaxzA/PuZ7b94AXYLCWMELoFiyjNT8jLiWXhd9dQQwjCAEDCqqyeKynKQWfNfOsO/g4fV2jN91XX4IXRzmYvtPr/Xe89rj3/6O8Gc1U6X6QSmpzLAAT4RY3rUY5z3ifoOGIj7irLf4jHOmxdVmz3IqQXxD3KfmlSzxM3E406brbVws7KhfM/DpBa20skR1gHIIs5hDko4IGRJimMAUFmhH/3smm54EtsFQhoEN6MjDInecFIYCNOJFlKBiHGFiCRFKie/69w5dLXsAREfpKb+rbb4A1RMgeOVqI1v0nWngMs0UQ/nZrMf2mrmo1OJABeg5dpy3VcAXAho1x3mvOE7jFOi+J6/9CQKfZRsiTTfTAAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAASZoAMABAAAAAEAAAEdAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdAJw+2YAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjI4NTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xMTc3PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CjP0I9MAAEAASURBVHgB7J0FvBtF14entLi1OC9WoHjRj1K8WHF3p7y4S3Eo7l6KOxR3d3d9sWLFihSXogVa2G+euZ29m9xNskk2uZvkf35Nszs7O/Js7u7smXPOdAmsGIkIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIVEFgvCrO1akiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIi4AhIyaQfggiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIQNUEpGSqGqEKEAEREAEREAEREAEREAEREAEREAEREAERkJJJvwEREAEREAEREAEREAEREAEREAEREAEREIGqCUjJVDVCFSACIiACIiACIiACIiACIiACIiACIiACIiAlk34DIiACIiACIiACIiACIiACIiACIiACIiACVROQkqlqhCpABERABERABERABERABERABERABERABESgW7UIfv/9DxMEQbXF6HwREAEREAEREAERaBoCE040oRm/W9XDrJryGDt2rBkzZmx7HV3aNtu+cnbMuD2boT09Lo0S4tLb08blsAm5aZzYpWOaTbHJHaRLXGKHXEoQAREQAREQARGoN4GqRz8jv/ra/Pvvv/Vut+oTAREQAREQAREQgcwSmHOOnpltm2/Yb7//br7+5ju/2xTfccqnNn1UrqbK7eUpqtrSwNCetz2tPT08GjmYm9aWN0xrP9WVHaZHNiKb5HaSo3Ibl6HtK2cnv7Wu+WF5/qhNaE+j+PaEuPTSaW1l5CFsa3dMYux1cUV0qGlcQ0unuxzt/7XV3dYst+17HJYU2YhshnnDtEgZcdeg7fC43JGT2jZz0wun5VQSe20iRdvMdq/tHydGpEC6vQa557edEncdIoVpUwREQARSIdDFWiFVZYYkBVMq10GFiIAIiIAIiIAINBmB8cbLdlQCLJncILD9P3cFcnfbholt/9vDkY3I5rgrF7QfJmVchpzU8CQOhxnGne9PyU1v24tLy6nElzauLJu/7V9Ytk8YV1LpdDtE7pjXlhIzdI5Nc83LLcHttf/n2tC+W05eV3j7+e27divS7rDIuLScnA5PezHjTgzPT563/RS71aGY9oT2fONqbc9Owjix7c7N6NLjeJMxJqsvSN8FCMQpntr0hLlqKrfX/p8rrX23PW+41X4wLy+7bblys0TS2rO4vG1HwtM4aksIU31x0VITpLWVMq6w6Jcr3RffXsu4LfvVnjaujA5pbekd9a221bknt9Ubl0grYtLbknILcXt5edvSxrXP1TKu3eGpbRu5u3Fp7WXk5m1L75hGPeNSw4ORtPbixuXKzdt+Snt6XFqkGLsZl3dcDnuo/XzSrFhWHdJsSh7CcVk75nQH9F9RAlUrmYqWroMiIAIiIAIiIAIiIAIiIAIi0EkE4hRSbYqrXJWU28vTaLWl0fD2vO1p7enh0cjB3LRCedvSO+aNKPXCg5G09uLGtWxcptwvm6s9ob2Y9rRIMW15cw9x2IpNbPvXtuv/j1XqJVcKJr0GrgWuXeMaN67+9qT29IJprpDwP08lxMNGWEq4kZNaUd62osYVmPOVs9Net9+yh8flaO9th7Rx/emQ3paQe77NG3u9SO6Qc1yd+ipEIKmij/ML53VHwyqcKqv9P5eeu9um7Gr7n4L9qRGFWZjG4ZydcZkjqTmHc3ba6/Zl2MPdunY1M0w/na+05HfV7nIla1AGERABERABERABERABERABEegEAkmtQTqhaapSBDJJIE7x5FRReQqptjS6kKuoiksvntZeRlhSZCOyOY5XRCnYfqptRZgzp0lheofD4xJyvnJ2IiW2p4/bCttCptJpbQ3NQ9hWRqwC0LY6t1CXN/bauHy5mduT2tMLprU1zf4fl7ft4EQ2zmQ5IkumcmgprwiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIQCyBbAcLiG2yEkVABERABERABERABERABERABERABERABLJGQEqmrF0RtUcEREAEREAEREAEREAEREAEREAEREAEGpCAlEwNeNHUZBEQAREQAREQAREQAREQAREQAREQARHIGgEpmbJ2RdQeERABERABERABERABERABERABERABEWhAAlIyNeBFU5NFQAREQAREQAREQAREQAREQAREQAREIGsEpGTK2hVRe0RABERABERABERABERABERABERABESgAQlIydSAF01NFgEREAEREAEREAEREAEREAEREAEREIGsEZCSKWtXRO0RAREQAREQAREQAREQAREQAREQAREQgQYkICVTA140NVkEREAEREAEREAEREAEREAEREAEREAEskagW5oN+t///mcuueSSnCKnmGIK85///MesssoqZoEFFsg5dv3115unnnrK9O3b1wwYMCDnWKmdP/74w/z5559mqqmmKpr11VdfNZdeeqnp0aOHOfHEE13egQMHGs7fc889O7SpaGFFDiZtT5Ei3KGXXnrJXHHFFWaaaaYxxx13XNHst9xyi+P3zjvvmEknndTMPffcZvfddzezzz57h/M+//xzM8sss3RIV4IIiIAIiIAIiEBrErj88svNyy+/XLTzK620ktlkk02K5il2MG4cViy/P/brr7+agw46yO0yHmJcVGv57rvvzJFHHumqOfTQQ82ss86aepVBEJgvv/zSzDTTTKmXrQJFQAREQAREIBME7MMuNbn55psD26nYT7du3YIDDjgg+Pfff8P6dtttN5d3m222CdOSbFjlVDDzzDMHTz75ZMnsN954o6vDKljCvFbh5NLuvffeMK2ajXLaU6qea665xrVtjjnmKJgVhv/9739dvnze448/fvDAAw+E59qBTLD11lsHdpAYpmlDBERABERABERABLbYYovYsUR0bLHvvvtWBSpuHJakwG+++SZs20cffZTklKrzfPDBB2GdVjlWdXn5BViFXrDkkksGxxxzTP4h7YuACIiACIhA0xBI1ZIpqjW78sorjaVkmIm64447zGOPPWZOP/10ZznkrZY22GAD07Nnz7KtiazSxPzzzz/R6gpuL7TQQuaUU04xWFTVSsppTxpteOWVVwyzj127dnXWTiussIL55JNPzNFHH23sAMnsuOOOBssl5LLLLjNWcWVWXHHFNKpWGSIgAiIgAiIgAk1CwE72mdVXX9315uOPPzZW+eG2zzjjjNByaP7556+qt5WOwyabbDI3fqPyqaeeuqo2ZOXkQw45xLzwwgtmjTXWyEqT1A4REAEREAERSJ1AzZRM1jrJjDdeW8invfbay2y55ZYG9zgesChlrGWTmWSSSZwbG65eXnA7u+GGG8zw4cOdkspa9Dgzbe8Whzseyivk7rvvNj///LNZZ511zFVXXWX+/vtvs9FGGzmlCtubbrqpmWCCCTrU4evi+5FHHjGPP/64mW666czGG28cmi+jHKMdCH2ZaKKJ3Pb9999vvvjiC7PYYouZ//u//3PugXHtIfPIkSOdgu2zzz4z8803n1l//fVN9+7dXTn+P5RDt99+u/npp5/Mqquu6pOLfj///PPuOKbWBx98sOO81FJLmcknn9wce+yxznXuxx9/dIonFFIIptmwW2uttZz7Imn08ZlnnjHPPvusY9SvXz+z+OKLc8jJsGHDDHXNNddcrmza2adPH7Peeuu54xyD36hRowzXCR52hm7c2W1fn376qbn11lvdderfv79z6bvzzjud0m+zzTYL8+Jq+fDDD5vffvvNLLrooo6V//2EmbQhAiIgAiIgAiKQGoHlllvO8EFw1/dKpg033NBNAvqKCo2xcCdjHGatlcyHH37oxh/zzjuv4Xnvx21x47CbbrrJnce4iDGVtcB2k4eM5xZeeGFXLWMAQh0gfjzw9NNPm/fee8+FWaB8xhOMb5ZddtkOipvff//djTtpF+MTxoeEI7DW4GarrbZyY1BXeIn/ktbJWPCee+4xuAf+8ssvZsYZZzRMpvbq1cvVwDHGhQhjM8ZkTAp26dKlJEP6wFiVsAfLL7+8ue222wxjNFgzvoaxF9rB+JhxFeNCrsWCCy7o6vF5Ro8ebaz3gXn77bddmSgafTt9Hn2LgAiIgAiIQMUE7MMoNYm6y1lLo5xyrTIjNEG2D0t3LN9dzvrCB9ayKcxnO+W27YMvsMoH52rn0/y3HVi4srwLnH2Yhuc/9NBDQZyZts9rY0GFeSnPzpQFL774oisP02xfBybbXuyD2KUfddRRRdtjlS+Br8eXYwdjwRtvvOGLCh599NHADgDCeuxAI1hiiSXcfjF3OTvQCM+xCpnAzjgGdqAQlus3bFyBMJ9vA+1CMAm3sbJyjlO/VQiGLo2DBw92x2mTtQRz20svvbQ7/7rrrss515d/2mmnueP8Rzv9eRynfKv4c+dxTb2ceuqpgbXKyinPKrwCq7zyWfQtAiIgAiIgAiJQQwKMf/yz3E6A5dTkxzP5Yyw7oRf07t07PM+fzzP+q6++cmXEjcPs5JU7xypZcp7/uPw/+OCD7rw4d7mddtrJnWcVRoGN0ZRTL2MeL9Q9zzzz5By3Cp+A8mmjVWz5rDnfce5ySeokjIFVkOXURz0TTjhh8Oabb7o6rHKow/GxY8cGSRhee+217lw7ERhYhVlOOZTrhbGytVzPOU471l57bZ8lsBOfgbVOy8kz8cQTB4zrJCIgAiIgAiKQBoG6rS4322yz2edcm1gFjt/M+cbFbsSIEc7aBncvrHxwBWOWhxkgvsnjZ7QIynj44YfnlIFlzqBBg9zsEMEqiwmuZRdccIErm5mzH374wQXOLnZO9Fih9jBDtP322zvrJL6x+Nl2220NFk1YdSF2YGEwU8eaiMDnViFmrILGvPvuu9EqYrdhYmMyuWOvvfaaIZA5QdVhzCwkZSPMoDEbiTDbBTvM3qkTNsz+Ybk0dOhQ1y47GDJDhgzpELyd2U2si+BqYzO4gOtYTCHMxFnFmdlhhx3cvlVMuW/cGXfddVc3m4eVFVZKuEved9997rj/zw6+DNeRmUXM85944gnHw8bbMmeeeabPpm8REAEREAEREIFOJpA/xrr66qvduAVLIix4rILIWc9geWPjXpZsLRbuVoHirKiwmh4zZoy58MILS56HhTTjqddffz0c5zB+8XLEEUeY999/31mnM8bBcuqtt95y5fs85X4Xq5O+Yz3EQjeM47799ls3XsLaipAFCGMd73rI+MyPZ8thiAUUi7vw7ReHYQEdxlLI8ccf7yyesHRn0Rss0LHUZwwNZ2T//fc3LBhjJw3dmIuxGeNWxqRY1UtEQAREQAREoGoCaWiqfBnFLJms+1Y4a+JnS/ItmaxSweWxfvgBs1vWXS2wih9ffPjtrV6igb/9LFt+gMq4GTSfl/q9RGfwrDlzkMSSyZ+b3x6rSHH9sC52gTWZdtnov59BY6YMyyN78dwHix8v9uHv0opZMvm8dvAQWNe1gBkoXxbfa665ZmAHai6bHYS4Y8xsebEDIZdmlXVB1ErLKsRcunXtc1m9JRPtxsosX+wgyiVZl7gAyy7qps9ItH9WceTS+G/zzTd3+bwlk105xu0vs8wyYR47GHJpdpAUpjXqBtcWNv4TvdaN2ie1WwREQAREoPkIRMdBhSyZ8sdYULAr/bqxjlVUBFbhEdiVbt0z/OSTT3aQ4sZh3pLJThSGIK2CxJ1nJ6ZcWjFLpuiYIWopT1uQaaed1pVlY1W6ff6zyiuXxlilEkumYnVa1zRXNhbbjMvsBKYbR4aVj9tYeeWVXb78wN+lGHpLJsZj1g3QlYbHgHWTc+V56y8WuaF/UasurOatW6HrM+fSRvLYyb+weVhIkWYnDsM0bYiACIiACIhApQTGsw+Vuoj3Q6cy4vfECdY+WOQQl4cZGKuQcDMw6667bhjIOu68aBp+90mFGERe/OwS+1hTRcXCDXeTBBzHQgphRo6+2sGOi0Xkz8WiyddBbCo7cAnLj7YjTCywQSwDgqoz82QHEeFsHtZCWB8VEmIwIcRXYobLizWndpvEw6LtXoi5kL90MDOEzBQSH8BbUJHfsyIWE0JwciyZvGCFFRXPillAOPHhd4BYc/fQKit6Tta3scayLobO8o6A68wsYvmFlRn7WMDxsYPfrHdF7RMBERABERCBkED+GItnPguRME5jgRXiBTGGQPx4IDw5ZqOnXfzFi1+g5a+//vJJBb/jziMz8TiJ7Wknxty5WFh5yR9/+PSk38XqXGSRRVxsJPpMnCisguacc05nBU5Mp2JSDkPGbMQzRbDqJzg6AjOrqHLxrdj3cbbYxnKd64N1ExZm/roQF9OPu4jvhDA+lYiACIiACIhAtQRqFvg7v2F22VaXxEMRk+g4YfUQTJ/vuusuZ2aN4gRlBSbI008/fQc3rrgy/AM37lh+mleEkB514bOWTjlZowoXTIpLCW5nCG057LDDOmRHMYMpNYJrG4MhTKwRr3RxOwX+I6A55tEE4L7ooosM9TGI4INpNkonOyPpTKHjiph55pldclTxR4IfXBAo0s6WhafmM7Wzi65sgosTcNIugezaQOByri+Cex6CYo0Bp43Z4PYx0Y6KZ4VykUCc+QJ7FHGNIgcccIBz+4u21ysXo2lso3SCJS6bEhEQAREQARHIOoH88QBu9CeccIJhMuqkk05ywbdxx8Jtzo8HivUpGrA6SX5fVrHzUMLQHsY0uK5Z6yF3Wv74w5eV9LtYnUwc4Y625557mltuucWtqEwoAca0BPcmWDl54qQchtE2UFaUGccYU6FsQtnmhVAQCGNsP+ZiH0VY/qp9Pug6xyUiIAIiIAIiUCmBNo1ApWcnOA8lCj781l3I5bYBFMMVR/JPxzd/9913dy/el112mRlhLYrOOussly3q2+8fslHljy/LH/P7xb5ZzYRYQIgv37qeOX93lFp+QOCVUd9//71biYP8fiaIbV+nb49/SLPiCit27Lfffm7lOpQtKG9QKNmAlM7Kh/NZoQ1hUODb4RIK/MeKJSgnWCWF2EW0BUUGAxzvT08MJSS/baR5yyJWdGEgiLAKCzELkKhlFfu+DLYR6kTBxAwYsQawgCK+E+IVKsQM8MpE4jbRN6yfiA8QFZY2Rjh/n332cayw5qJtM8wwg+F6NIJgvQRz4kqVI8Se4HfG+RIREAEREAERyDKB/PEAq5wh1vXdxYe0rnLhpJ0fD3RGf1ZbbTVXrXXVc/E2UbRcfPHFNWvKCy+84MYwNhyEGwcQL5PV3RDGfn4C0fPz40WOp8UQhZOf0GM85/mjTMIafe+99zY9reWYtxjDsonxKemMcxmncVwiAiIgAiIgAtUSqJmJiH+IYbbsFTIoDU488cSCbeY4bnLMtDADhKKHJVaRVVZZJTwPax2scAhCjek2wRMrEQJyo3Dp3r27C7xNGSg6bFwhVxymzpgWb7PNNi4YOUopZojyJa49mIxjbYRLHibJKBEwR2abma5JJ53UzW5hibTLLru4gNjPPfdcaOGUX0d0n8EcAxmUXitY9zOUVlhYeQUTihtvIk7bECybUHgdfPDBYZsYfJGGWTXL2KI4ou+lFCUoobAuwgKL4N7w84M3Bilcc2YSsXDCTeycc85x7WWQl28ltvPOO7uA57jGMTjCGoq+UXYjWfgQfN0H3oxeq6TbuNH5v5Ok5yifCIiACIiACHQmAcYgWAudffbZLtQB4yTGTYgfk3RG+7D4ZrKSSU5c+nm++rFdLdpDaAQbR9SN4Vi4hnAEfvKISTcfJsKPyZh8o20E5E6TIW74TPxdddVVBjc9xtGMdRmTMXZkfM3YmbEZCiaUY4y37r//fmNjZZo99tijFnhUpgiIgAiIQKsRsA/e1CQa+NtydEEEretVYF2hAgJaE/w6KgTeJp9V4oTJVukSWAVJGCSboNoDBgwIrFVQmMeuhBEGLiTAJOKDeVtLoDAfG3EBJ6N5rdWMawP1WGWPC2DpC7BxjQKrwHHH7QPa9cHO+Lh9Ajl7iWuPXcHNBbm2D3SX3w4s3P6oUaP8aS54I323s08uj7WECayZtdu2A5IwX9wGS+JahUzIAY70y7qcBdTt5euvvw4Iss1x6rGrrLhD1sLM9cf3z1rTBHZQFFjTbn9q4AN/k54vdiW8wAeYnHLKKV1ATQJ1U48drLjsLOlLcEsCfFoFYmAHLy6oJHmstVdYpJ3lC2wMo5CDHZAF1s0wPJ71DRubyvWbflX64drAQCICIiACIiACnUEgSeDv/DGWVSgFdrIqsBNP7vnXv3//wD8T7cq5rhtx4zAf+NsqQ8Kunnvuua6MxRZbzKUVC/zNQiVe7ARe+Oz1i61wzFoIBQS0thNhwVprrRXYibwwn1WA+dNzvq3FUZjHxop0x6wFvksrVScLe1gFT2AnWcMy7IRjYN30wjqs4ilgXMxYgfHhxx9/HCRh6AN/Mz6KirVQcmXZMBNhMnlt7CaXbi2nAjtJG9iV8cLjjM1YFAYutIMFW7huBC+XiIAIiIAIiEAaBLpQiH3IZE5w38K6hfhFzLzki1XWGNzRmKGqVnDNwsIG66I4wYwYa6FonKL8fIXag7sgcQEwQcaUOU6sUsjwoY5yBcsq2oflGG50cYJL4AjresiMVlwfaR/WSN76LK6MuDR+OtRN7IP8vtFvlg3m+mCh5AOMY9WEtZgd8LkZvGi5WEBhnUV5jSLMVGKFlJbYQaqb1UyrPJUjAiIgAiIgArUmgKUy4wDGUlkQFkXBNQ33PSx0ECzkCdDNOIgFZmoluKlZ5ZFz+SfGZb4QpBtrJ8aF0ZiTaTPE4p9xXVwbfJsY/xH6oFFCE/h261sEREAERCDbBDKrZMo2NrUuCQH8/Ym1RBDwQw891JnOn3nmmU7phkm3j9OVpKys5rHLLxu7THMY26vadtpZV+OD5Fdbls4XAREQAREQgVYkwEIiuN6jPLGW1+6b0Aq4kK1gXfyY0JGIgAiIgAiIgAjUhoCUTLXhqlItAWINbLvtth3iMmD5w4qBcVZVjQaOVfGqXbEmv88ZNS7Mb6b2RUAEREAERCCTBLDeJvg3QbejQmwkrJwWXHDBaLK2RUAEREAEREAEUiQgJVOKMFVURwK4PRLQHNdHAk+y4pxffa9j7sZL8SsQptlyucylSVNliYAIiIAItCIBQgUQ2BqXMFzYcE8jILdf4a0VmajPIiACIiACIlAPAlIy1YOy6mhKAmnHY/KQpGTyJPQtAiIgAiIgAiIgAiIgAiIgAiLQSATiI1E3Ug/UVhFoIgIssYzySiICIiACIiACIiACIiACIiACIiACjUZASqZGu2Jqb1MTYLVAiQiIgAiIgAiIgAiIgAiIgAiIgAg0IgG5yzXiVVObM0OgFjGZFPg7M5dXDREBERABERABERABERABERABESiDgCyZyoClrCKQT2DRRRfNT9K+CIiACIiACIiACIiACIiACIiACLQkASmZWvKyq9NpEVh33XXTKsqMN9545uCDD06tPBUkAiIgAiIgAiIgAiIgAiIgAiIgAvUkIHe5etJWXU1JYIEFFjDvvPNOKn2Tq1wqGFWICIiACIiACIiACIiACIiACIhAJxCQJVMnQFeVzUXgvPPOS6VDRx11VCrlqBAREAEREAEREAEREAERaCYC//zzj+GTPyH777//Ju7m6NGjE+f1Gcspn/aNGTPGn5r4O2kdSfPlV5zPLP941vbL6Wc+80K/k6z1sdnbIyVTs19h9a/mBFZYYQVTrYJo4MCB5uijj655W1WBCIiACIiACIiACIiACDQagVlnndVMP/305vjjj3dNv+mmm8zCCy9spplmGrPZZpuZP/74o2iXnnrqKUMZSeXXX381W2yxhZlhhhnMHHPMYe69996ip6IYWWeddUw5k8+DBw82eETMNNNMZsCAAabQKtN4TCy55JKmR48epm/fvuaRRx4p2hZ/8P333zfbbLONmWWWWcwSSyxhLrjgAn+owzcMZ5555vCzyiqrdMhT64RqmXMN6MNkk01mhgwZUuvmqvwiBKRkKgJHh0QgKQEURJUqmg466CBz+umnJ61K+URABERABERABERABESg5Qjcd999ZtCgQeaTTz4x++yzj1PovPfee+b33383hx9+eEEe11xzjVl//fXLsjI65JBDzAQTTGA++ugjc8YZZziF07fffhtbx1dffWU22GADc//998cej0v83//+Z04++WTz4IMPmg8++MB8/vnn5qSTTuqQFcXJRhttZFhsaOTIkeacc85xiiPaVUoOPfRQM9tss7k+XHvttebII480w4YNiz3tscceM5dccol56KGH3Oeiiy6KzVfLxGqZE9+Wa7HmmmvWspkqOwEBKZkSQFIWEUhCAEXT448/bhZaaKGS2SeffHKXh/ynnHJKyfzKIAIiIAIiIAIiIAIiIAIiYAxWTP379zfLLrusmW666cwxxxxjhg4dGovmhhtucMqbM888M/Z4ocQrrrjCTSAzZkeB1K9fP1dvXP7VVlvNzD333Ga99daLOxyb1rVrV3P11VeHljcokYYPH94hL8ok0lGuYaGDJdPSSy9d0rLqr7/+cpZPeEtMOOGEZq655jIrrbSSueWWWzrUgfIKKyr6Mf/887vPnHPO2SFfrRNqzbzW7Vf57QSkZGpnoS0RqJoArnNvvPGGeygxwxInffr0Mfvvv7/zKSe/RAREQAREQAREQAREQAREIBmBESNGOKWJz92rVy/zww8/mN9++80nhd+rrrqqeeutt5y7WJhYYuObb74xf//9t5l99tnDnNRBvXGCFdBpp53mLJ/ijsel4eqHogwFz9lnn21Qhu22224dsuIOONFEE5kuXbqEx3ANjFNIhRnsBoqlyy67zCmaSMcVDaup//u//4tmc9uvvfaamXLKKZ21Fko7lE0ffvhhh3y1TKgH81q2X2XnEpCSKZeH9kQgFQJYNfHAuP7661153OgxiSXw3ksvvWQ4LhEBERABERABERABERABESiPAIqZ7t27hydNMcUUbhtFU75MNdVUBjeqcoTyUbpEFTvUEVc+5aIIqlSeffZZgxsgiiSCVucLcZhQ+jBBjWsg1k+c8/333+dnLbiPUmrDDTc0KNyIG5UvP//8s7PEOuKII1zZsF133XXN2LFj87PWbL+ezGvWCRUcEijvLy48TRsiIAKlCOD/TcDAXXfd1RA879VXXy11io6LgAiIgAiIgAiIgAiIgAgUIYDiJWq1xDYKIQKDpyH55VMmdRAEPG3ZdNNNXQwkApoTfDtuZTVit44aNcopiR599FGz9957u1hLSdqCMmrFFVc00047rSEuU5xstdVWBmusBRdc0FmInXrqqU6hReDwekk9mderT61cj5RMrXz11feaEbjqqqtcUL499tjDreSAaSoB/iQiIAIiIAIiIAIiIAIiIAKVE2C1tE8//TQsgG0UTFgDpSEzzjijsyr6+uuvw+Koo2fPnuF+tRsPP/ywU+z4coj59N1335kvvvjCJ+V8s7rdZ599ZnjHIPB5kphJWActv/zybmU6Jr/HH3/8nDL9zhNPPGGefvppv+tc7Qh6npbSLiy4yEY9mBepXodSJiAlU8pAVZwIXH755W4ZUmIynXvuuQ4ISiYC9/HwkIiACIiACIiACIiACIiACFRGYMsttzS33367i7U0evRog+XNJpts4goj4DVWObiIlZJ33nnHvPvuux2yoaxiRbcTTzzRxWZ68cUXnRJmjTXWcHlxW3v77bc7nJefQDBt2sJ3vowZM8bsvPPObmU83OSGDBniFEco0PL7sPXWW4cBu4kvhbucDzJeqA9YROEihxUTgdF/+eUXZw0FLyTaB45tt912Lm4TLnKE/EDphRtgsT5QziuvvJITq4pYUbTRC6u9Pffcc37XFGpvWszDirTRqQS6dWrtqjyzBNBoI3wTnJqPpDQBlv7kgYHfNMudellsscXcJi5zq6++uk/WtwiIgAiIgAiIgAiIgAiIQBkEWAFtv/32c8G8p556atO7d29z/vnnuxJwD1t55ZWdEmWeeeYpWqpf4RnroHwhliqKHKyXWAmOcf2ss87qsrGN+5yPvZp/rt9HwUJbPv7445wg4hxfc801w1WpUeTgLnbbbbc5t7/8PhBUHO+IY4891nTr1s3wvuGtjAr1Abc64sDy8Wyol+Di7Ef7QJwmVrwmGDkKLoJ/s4IfUqwPHN9rr71cnKfDDjuMXXPOOee4c2699Va3T7yp4447LlREFWovmdNg7irVf51OoIsNRBx0eivUgEwQIBj13XffXdCta5FFFnE3WwWtjr9cF1xwgdl9993NgQce6GZU8nMxM0F8psMPPzz/kPZFQAREQAREQAREQAREoOEI1GtieqaZZnLWS0sssUTICIUIljnRIODhwQQbuJ9ddNFF5oQTTiiY+9tvv3XxjKJBwAtmjjnA5PPgwYPNxBNPHHPUuODaBBT3SqPYTOMSf/zxR0Mg86gk6UM0f7FtLKp++umnDoHMS/WhWJn5x5K0t1rmWKFhiUXsKknnEJC7XOdwz1StPBxQIGFKWSxu0Ouvv+7yLLTQQs7CKVOd6OTG4BaHgunQQw+NVTDRPFzmFPy7ky+UqhcBERABERABERABEaiKABPOuGGheOHbu2T5tEX7LG1qMSmNdczzzz8ftp3VmytVMFEIcYiw7CkmWPVUqmDC2ol3rEIKJurFMimJgom8+Qom0pL0gXxJBIut/JXykvQhSdk+T5L2Vsoc2xmstLAck3QuAVkydS7/Tq+dBwDKpUrkqKOOqskDpJK2dOY5zE7su+++hmU/MQctJBy77LLLQnPRQvmULgIiIAIiIAIiIAIiIAJZI8DENO8NfJueaxnTY+62JnYf9z1quDE/2Q8y4l6T5rsC7l2//vqrWXrppd0qa22V6H8RaCeAksm/1xI/q2/fvu0HtVVXAlIy1RV3tiqrRsHke4L/bivHazrzzDPNwIEDEz1EWRVi7bXXNl9++aVhBQWJCIiACIiACIiACIiACDQCgfC9AYXS7FbB5BVLhRpvFU7dv3vYjPpiWKJxcqFilC4CItB4BKRkarxrllqLKzX9zG9Aq4b1YiWLgw8+2AXhGzRoUD6WDvssg4pyibhXKJskIiACIiACIiACIiACIpB1AqGCCeul2cscw35yj7NqavWJ6axfY7VPBNIkoJhMadJsoLIOOeSQVFqLogqz1VaTk046ySmYWNo0iYIJPjPMMINboUJxmVrt16L+ioAIiIAIiIAIiEDjEnAuSJUomOgySilr9eTdmBqXglouAiKQlICUTElJNVE+/Kj98pHVdgsrJgLwOd/sagtrkPOPP/54wzKdMCTQdzmi4N/l0FJeERABERABERABERCBziSAFVP3mXuXb8EUbbR1r+NdgbIkIiACzU9ASqbmv8YdethKCqEOna8ygYcjlkunn366Oeigg8ouDSVTsRX8yi5QJ4iACIiACIiACIiACIhADQgw7sUCadS0/asrnfhNi+7XHjS8utJ0tgiIQMYJSMmU8QtUi+b98ccfqReblvtd6g1LsUCUSzxozzrrLBfsu5KiF1tsMTNy5Ejz+eefV3K6zhEBERABERABERABERCB+hIoFeQ7SWvSKCNJPcojAiLQ6QSkZOr0S1D/Bjz22GOpVzp69OjUy8xSgbjH4SY3ZMgQs++++1bcNCyZEMVlqhihThQBERABERABERABEagXAWIxpSS43cmjIiWYKkYEMkxASqYMX5xaNa0WCo4333yzVs3t9HJZQY5A3+eff77Zc889q2rPNNNMY3r16iUlU1UUdbIIiIAIiIAIiIAIiECtCaQdrHvUb3+bJ598stbNVvkiIAKdTEBKpk6+AJ1RvbemSbPuhRdeOM3iMlPWwIEDzamnnmouuugis9tuu6XSLsVlSgWjChEBERABERABERABEag1gR42nlJa0n0uM+q3v9IqTeWIgAhklICUTHkXphVMOFdZZZW8Xle/u+KKK1ZfSMZKwC3uzDPPNJdeeqnZeeedU2sdSqZaWJOl1kAVJAIiIAIiIAIiIAIi0PIEFll8KWN+Gp4qh/XWWjXV8lSYCIhA9gi0vJIJpdJSSy1l5p9/ftOlSxeDsoRvPosvvnhTLrW5+uqrp/5LnHLKKVMvszML3GuvvczgwYPNlVdeaXbYYYdUm0Lw72+++cZ88sknqZarwqonwCoq/u9f3233QXEQB/0G2n8D1d9lVIIIiIAINA6B7pNNaMyoD9Jr8Ih70ytLJYmACGSWQLfMtqzGDUO5tP/++5vXXnutYE1Ym/DBH/moo45qSoVTwc6XeWCFFVYo84zsZt99993NBRdcYIYOHWq23nrr1Bvq3RX5bc0+++ypl68CKyfA3zq/5X79+lVeiM4UARFoSgLcHxg7NNPzrikvlDolAiKQGgHGQ69/eKsZlVqJRvfQFFmqKBHIKoGWVDJhrVBuIDvy//TTT866JasXM2m7GCAfccQRbrW0pOcUy3fggQc2zQMDt7hLLrnEXHfddWaLLbYo1u2Kj3Xv3t3MO++8ToG58cYbV1yOTqwNAQZU3CMkIiACIuAJoFwqd9zgz9W3CIiACDQqAd4Z3L1vWusy133u6rrxyT3ufMqUiIAINDeB8Zq7ex17V4mCyZdyzjnnmCWWWMLvNvT3cccd51wE0+gEgbGbQXCLQ8F044031kzB5Dkp+LcnoW8REAEREAEREAEREIEsEkAh5JRCn6Tg5mZd5fAMkYiACDQ/gZZTMlU7E/nyyy+bQYMGNcUv47zzzqu6H83ysBgwYIC5/PLLza233mo23XTTqrmUKkDBv0sR0nEREAEREAEREAEREIHOJvD444/buEzWkmmcJVJF7bHnoqySpXhF9HSSCDQcgZZSMu20006pXKDjjz/exWVIpbBOLISbfTVKomaJU7XNNtuYq666ytxxxx1mww03rMsVIfj3Dz/8YD74IMVginVpuSoRAREQAREQAREQARFoJQJO0UTQ7koUTZwjK6ZW+rmoryJgWkbJhOacpejTkkMOOSStojq1HLgEQWB22WWXxO1YZJFFDA+bZpiNIO7Stddea+6++26z3nrrJWZQbcZo8O9qy9L5IiACIiACIiACIiACIlArAuHEdDmKJmv91P0D6zVhz+G9gTIkIiACrUGgZZRMf/zxR6pX9MUXX0y1vM4u7MILL3RWTX379i3YFBQjWC+xIl8zPChwi7vlllvMvffea9Zee+2C/a7Fgckmm8z07t3bBf+uRfkqUwREQAREQAREQAREQATSIsDkslMW9fzdmMd3a7NqwkoJVzovbPN57Sz3WaTXNG4yuxneG3wX9S0CIlCaQLfSWZojx3333Zd6R5ptKWMeHlHrpGWXXdZ8+eWXLlZRsz0cNtpoI6dc4nfRv3//1H8bSQpU8O8klJRHBERABERABERABEQgCwR4H+DD+8KTTz7ZFj4E66Y8Ic9RR13k8uYd0q4IiEALEGgZS6a333479cuJkqmZZcsttzQ//vhj0z0g1l9/ffPggw+azlQw8bshLtOrr77azD8h9U0EREAEREAEREAERKDJCKBkwqrpn3/+MUsuuaTp2rWrmXvuuV0aYTjkHtdkF1zdEYEyCbSEkqlWyiA0+M0ss846q/n555/NTz/91BTd5KGHWxwPPlzkVlpppU7tF5ZM8H333Xc7tR2qXAREQAREQAREQAREQATKITBmzBiz5pprmuHDh5u99trLfPvtt003MV0OD+UVARFoJ9ASSiZMNmsh/fr1q0WxmSlzlllmcW35/PPPM9OmShsyduxY9yB87rnnnAVTFq6dgn9XejV1ngiIgAiIgAiIgAiIQGcRGD16tBtX/+9//3Pj6o033tiMGjXKvPfee53VJNUrAiKQIQItoWTKEO+GagqWTMhnn33WUO3Ob+yff/7pHoSvvPKKexAus8wy+Vk6ZX+iiSYyrNQnl7lOwa9KRUAEREAEREAEREAEyiTw22+/mbXWWssQioTQEywapInTMiEquwg0OYGWUTItvvjiqV/KaJDs1AvPQIE9evQwU0wxhWlkS6bff//dKZjeeust9yDEbzxLouDfWboaaosIiIAIiIAIiIAIiEAhAoR5wEXugw8+cONq/37FxOnCCy+sidNC4JQuAi1GoGWUTGjcJ5100ha7vNV3F2umRrVk+uWXX9yDEF9xYjD16dOneiApl6Dg3ykDVXEiIAIiIAIiIAIiIAKpE2AxIBRMvBdgwYQ1flSYOJV1fpSItkWgdQm0jJKJuExYtaQlRx11VFpFZboc4jI1oiUTwcp5EI4YMcI9CFHmZFF4IPO7xNJKIgIiIAIiIAIiIAIiIAJZI0BQb8bVX3/9tRtXL7jggh2aKOv8DkiUIAItS6CllExpKYYWWmgh0+yucv4vohEtmb777jv3IPzyyy/dg5DrlVXhgdylSxfN/GT1AqldIiACIiACIiACItDCBL766is3rmYC9/777zfzzz9/LA3GtMRrGjZsWOxxJYqACLQOgZZRMnFJUQwRnA6/4Wrk0EMPreb0hjq30SyZmGFhpuWHH35wCqYFFlgg07y7devmgiWyOodEBERABERABERABERABLJC4IsvvnDj6j/++MONq+eee+6CTdPEaUE0OiACLUegpZRMXN0XXnjBxJl4Jr3yxHXaYostzOabb24efvjhpKc1bL5GsmQaOXKkexAyi0IMpnnnnbchuPNQlg97Q1yqxI1kGV9iFZx33nmJz4lm/Pvvv03Pnj3Dz0UXXRQefuONN8yBBx5o5ptvPrPSSiuZwYMHh8eSblTbPl9PNW25/vrrzXLLLWe4x6y99trmiiuu8MVW9F1NW26//XbTv39/M/vss5stt9zS3HDDDRW1Ie6k119/3cwxxxyG/iaRl19+2Wy44YaGgfxcc83ltl977bUkp2YiDxakp556qmGRBe5tAwcOTOyq/tJLL5l11lnH/e75bWB9/Ndff4X92muvvcK/iVVXXTVM14YIiIAIiED6BAg5wcTt2LFjnYJpzjnnLFoJE6eKNVoUkQ6KQMsQaDklE1eWgezhhx9e1kVmxYTHH3/cmYFedtllLk4Rg1wG0eeff775888/yyqvUTLzAvjvv/9mPvg3QQh5EPJyTjBCXs4aRfRAbpQrlaydvBSjhEbpUel9IQgC8+mnnzrlxP777x8GrSdt2WWXNU899ZTZaaedzGyzzWb23Xdfs88++yRrnM2VRvuorJq2oNRBmfPPP/+YPfbYw33vuOOOplIr0WracsoppzhFzq+//ura8v3337uJBBQl1QrXf6uttjKffPKJe3aUKu+xxx5zyplnnnnGLQ/NghVss2jBE088Uer0Tj/OiwgKQxSf6667rllttdXMxRdfbFZZZRUzevToou1DOUXsxHfeeccMGDDA9OrVy5xxxhlm9dVXD/+O4MHfw/jjj2/ILxEBERABEagNgY8++sg9h7p27erG1Ux8JRFNnCahpDwi0AIE7MtMy4pVGgVLLLFEYC9zzme88cbL2bezqbGM7IteYAfDLu9kk00W2Be94M0334zN26iJ9iHj+mdfdDLbBdpo3eICG3spsC+bmW1noYa98sorjrF1mSuURel1IsC9oNDfe5ImPPfcc4G1MApsnC13TU8//fQkp3XIY5UT7nz7Qp1zzCq2A2sVE9gX9jB90KBBrr73338/TCu0kVb7KL/StlhFhGNkXZdzmmmtdwI7CxrYmA856Ul2Km2LnaUNJpxwwsBatwZjxowJqzrrrLMCngOPPPJImFbJxt577x1MMskk7lpaZUvJInr37h1Ya9nArowZ5rXBVl0brTI6TMvqxoknnuh+i++9917YxEcffdT1/5JLLgnT4jZ23nlnx8oq+cLD55xzjjv3nnvuCdPY6Nevn7vn5yTWeIfxAvcHviUiIAIi0MwEGE9Ya9rAKowCq9Avq6vW8jqwYUnKOkeZRUAEmo9AS1oyed0hs6YvvviisZfVWSnZl0tnnm9f2ty+HUy6Y8RyihPM+XHxIA4QM/B33323Icg0s6233XZb3CkNl4YlE4KlUBblgw8+cBZM9kXRzbT49maxrYXaxKwPM/NymStEqDHSn332WWdlhKVQMdcoq1Q0t9xyi7GKhLI6Nnz4cPPQQw+ZbbbZJieu3K677uqCx2NhWUyStq9YGf5YNW355ptvnHXKcccd54tz394k/+23385JL7VTTVtgwvXaeuutDWb+XrCyIiB/NS58Dz74oHOXPPnkk32xRb9x88XyCXfsySefPMw77bTTOosgVqDEqjTLMmTIEGMVQGaeeeYJm4lLJ67LVskUpuVv8AyeaaaZDKymnnrq8DC/CQSXQ4kIiIAIiEDtCbz77rtuXN29e3c3rp5xxhnLqpQxLc8y3bfLwqbMItB0BNpH1U3XtfI6hMKJTyUy/fTTm8MOO8x9br75ZnP55ZebjTbayJn777DDDoYPLwqNKLx4Mfj//PPPM9d8O1vu4nf06NHD3HXXXWaGGWbIXBuTNoiHsoJ/J6WVzXyYlFvLJbPLLrsYlvotJMRpuvLKKw1KA2u5Uihbh3SUUwiLF0TlP//5j5l55pmNnXmMJnfYTtq+DifGJFTTFtp75pln5pSK8uSOO+4wxLxbfPHFc46V2qmmLQQ0RfLv/dNNN52hnb7sUm3IP87CA9tvv72LR7T88svnH47dt9awBqV5vlgLKxdLEAW6ta7KP5yZfdzXWIEIJWi+WIthc+edd+Ynh/so9I488shw32/4yRoUVxIREAEREIHaEmBcgqsziiUmzqNK/6Q1RydOiU0pEQERaE0C2R2xNuj12GSTTdzynmjw11hjDcNsPS8sKJqsq0pD9oqXm6xZMmHtwCz3NNNM42ZaGlnBxI+Ch7IsmRryzyNsNIGO99tvP2Pdo8K0uI2NN97YnHTSSQbldDniFVf85vNlqqmmchaV+enR/aTti55TaLvatvhyUfJgiUWsBxZlIGA/VonlSDVtISA3Qr1RYaCNYp3lmisR4mUxsZBvrVVJWddcc41TWm677baVnF63c0pdh59//rlkXCbfWCyDeTlh8ubss892FoL+mL5FQAREQATSJ8B7C54YrCpNbNNKFEy+VYo16knoWwRal4CUTDW69gQKt/EkzHfffee+sVJZZpllDLPaWDE0kvDAyZIlEwGVUTBhYcWDMO6lu5H40lb/QM66O0yjcc1iexnEHXLIIWVbN+JOhWC5ly+YtfuX/PxjtdhPqy0EeWZgS7BthADZuE6VI9W0BVcuBtK4GtrYQa5aFOrHHHOM2y4VrDqunViycl9COTTBBBPEZUmcRpt22203p4SuNCh64sqqzFjqOlA8z8NSgjIKV0Pc0LFwwjqKpbMlIiACIiACtSHAJCfjalaP4/nFmKIakXV+NfR0rgg0BwEpmWp8HbFqYNlllqC+//77nUsXbhRY3rDC3ccff1zjFlRffJYsmVDW8SBkqXGsD+JeuKvvcf1L4IHMSluyZqo/+0apEWslJO6F+/fff69aoVEOh7TawgqdWDChSCAOEvfGCy+8sJymmGrawrmXXnqps1hiBTRW68O6CQum9dZbz0wxxRRltYXVeFjp7/jjjzc2mHhZ5+ZnvuGGG9yssg2+6p4dxG3LspS6DrQ9idJtyimndK7DWLmhsGP1P34XEhEQAREQgfQJsOI24+r555/fKZhw3a5WvHV+uZNG1dar80VABLJDQEqmOl4LlmK+6aabDC8iuM8xe86swaabbmoeeOCBOrakvKqyYsn08ssvuwchQWSZaSn3BbC8Xtc3N5ZvE088sZRM9cXeULURIwj58ccfO7SbtGpnHjsUWiQh7bagWMB6iL8BYjOVI9W2Zf311zdYR9qV0Yxd4c4QV4/7C0zLjaVnVwM0dvU8Z1WGtRqfwYMHu+4QX4j9uOuX319cxFC6Lb300ubpp58uux355dVjv9R1oA1c56RCPEACsuPmyQQNAdolIiACIiAC6REgjAcKpkUXXdQ993gGpyEomXgWauI0DZoqQwQak4CUTJ1w3ZgpP+GEE5w7gF+djvhNxKBgdZ44S4VOaGZYJZZMBLL99ddfw7R6b2DtwIOQ1ft4ASwV96be7UujPpkXp0GxecsguDfy4Ycf5nSS+wVBl/nbqJdU0xasOlG25PcDJRnBRuOCXxfrVzVtoVzcvFCk44521llnmQ022MC57L355puGgNXlCINqVobDJRorHD5eafbMM8+4/VL3UZRtxPYigDaTD+UoZsppa9p5uX7MgOdfV+ohba655nJKxLh6cZek3yiT8mW++eZzzx5WJZSIgAiIgAikQ4AJDMbVLCbCuDqJpWnSmjVxmpSU8olA8xKQkqmTr+2AAQPMU089ZVhKGyXD3nvv7Wat+WZ2PQvCCxjSWXGZeDnjQdinTx/3ICw3MHAWGCZpg4/LlCSv8rQeAe4PuE5df/31OZ1n1S5iB2GFUy+ppi2//PKLc4HCcigqTz75pHMfLnd1uWragvLcuy5H23Ldddc5xUa5wbZx4SU2VvTz+OOPu6JZUY90XPIKyQUXXGCOPvpop2S66qqrUh30F6ozzfTNN9/crfQZnSjB7Y2XmWK/T5RTJ598snv+4TbsBQUglmUsnsFkh0QEREAERKB6Ak888YQbVxMnludWLVYu1Zi2+uukEkSgkQlIyZSRq4dbBO5zvIQMGjTIzWBj2YSLHYPszhQ/uO+MFeZ48UTBRNB0HoS4UDSr8LJMzKm///67WbuoflkCKBG41nEWH8UAEQT5gAMOMI899pjBLev99983uGDtueeeZrPNNnP3Cn/+scce6+oo1yqI84kVR/uK/b1X05Zll13WrLDCCi4wNu3EYggFDAr3SSed1Fl5+n7Uui0E/Saw9kUXXeTuvzBF0bPHHns4yybuO16StMXnTfLN/QzOXmnIvZ/V1GCAIp368j/eZYzrzrm33357yap4hvDJl0LpxMnKz5+0voEDBxrig7EENvcyXDHYZvXAI488MmxCft8nmmgic+CBB7q/if/+978uThcxqWgLiiYszCQiIAIiIALVE3jkkUfcuJo4hHfddVf1BRYogWcUzwGJCIhAaxJo3jf2Br2exADxsTwY2KN4ImYTLnbEceJT7tLn1aKgTbz4FHvprLaOuPN5mV5nnXVM//79Q5eTuHzNksYDGcGHfamllmqWbqkfeQQ+/fRTN/D6888/846U3t1pp51cUGoUVbx446LE38gZZ5yRc7KvI2pRkpOhyA4rvDEwLNW+StvStWtXpzgnQPZRRx3lPjQHC6ZbbrnFuVX55tW6LdRz0EEHuRXudtxxR1ct91oUdzCOStK2RM8ptk1sJjijXEJwFRs1apTbxqonTg4++GCngMK9jHOTrNaGEi9OiqXnz2onrY94efRju+22c0owgpWvvPLKLph71MU5v++0D978Ngj0ffXVV7sm4z6JEg4LKYkIiIAIiEB1BFi5E8U/n1pPYDOmZZXtMWPGmKwvXFEdVZ0tAiIQS8BG/pdknIB9GQjsC1lgY32wtndgB/CBdbGra6vty0NwxBFH1K1O+yAM7Gx+YF0s6lZnFiriGtu4XFloSku2gb8vq/jo9L5bBY/7W7cWS7FtsYO24L333gus1Vvs8WoTbfycwLq1JSqmmrZQx+uvvx5YJUbBuurVFhvzJxg+fHjBdnCgnLYULSiFg3bCIbAWQSmUlKyIcuuzK6cGdpW+ZIVHcllrreDtt98O7KRGJDV3s1+/fsECCyyQm1jjPev26P4m+ZaIgAiIQKMR4HlhJxACa/lcl6YPGzbM3TOff/75utSnSkRABLJFQO5ysaq3bCWyFDarDTFrfd5555m33nrL4EeNKweBZeshuMzVy5KJmXBc5FhC/NZbb61H9zJTh8yLM3MpMt0Q3EbnmWeemswO3njjjYZPGRLgAABAAElEQVRg2gSwTiLVtIU6CBCK21qc1LMtxP0hOHUhKbcthcpJI90O3g2x6upl8VhJfbPPPntFKx4SfJaltH0swDR4qQwREAERaGUCd999t1lrrbWcVSiuyPUQOxHgxhFaYa4etFWHCGSPgJRM2bsmBVtEnI7dd9/duVM99NBDbhCO+xwvR6yMVEn8lYKV5R1gwF+PwN/33HOPUzDhIshLXauJAiW22hUv3l/cx3AXvemmm4pnTPEof+sMSLMgakv8VcB9mXtljx494jOknFrv+go1n1VZ+XvIyqIYhdqpdBEQARHICgFi9+Eex4ql1157bV2bpTFtXXGrMhHIFAEpmTJ1OZI3hoE2sxEjRowwu+66q4thwcpTG220kVuBLXlJyXLWw5KJVbKIL7P11lsbVndqRcGSiVgplcTSaUVezdpnYtMQF4iBIXFuCln61KL/LEKAUiELorbEXwWshHr16hV/sAap9a6vUBdmmmkm9/fAM2LLLbcslE3pIiACIiAClgATVazsuf3224ex7uoJRtb59aStukQgWwS6Zas5ak25BFgOmxWa+AwdOtQFCsckFhc7rJxYqSep20uxumttyYRb3MYbb+wCxl555ZXFmtLUx3ggI5gXL7fcck3dV3WuMAFc0GxsrsIZdEQEWpAAKxDykYiACIiACBQngDcAiyawqMUll1xSPHONjjKmZfXY0aNHm4knnrhGtahYERCBLBKQJVMWr0qFbcIU9oknnjA2yJ7p27evW+YcVzqW4652GVEsmWyQYfPll19W2LrCp+EKhIIJpVgrK5ggRJwdXGCqvV6FaeuICIiACIiACIiACIhAsxLAGwAF0y677NJpCibYRidOm5W1+iUCIhBPQEqmeC4Nnbrkkku6hwqBwrFwevTRR92NftVVV604zhGWTEjacZlYntqudGF23nlnc+mllzY097Qaz0NZgRLToqlyREAEREAEREAERKA1CFx99dVmq622chPMF154Yad22k+cakzbqZdBlYtApxCQkqlTsNen0qmmmsoceOCBxi51bgj8R+BwZjaIr3Hcccclsko6+uijXRkE4kZ4cLGiERZTfKqRa665xsXV2G233cxFF11UTVFNda4CJTbV5VRnREAEREAEREAERKDmBK644goXdmLvvfc25557bs3rS1KBxrRJKCmPCDQfASmZmu+axvZo/fXXdytGvf322y4I4BlnnGEIorrtttuaJ598ssM5KJe6dOlijjnmGHP66aeHq/l89NFH5oUXXjArrrii+2A1VYmyCbc43PsIbnz++ed3qL+VE7Bkeuedd8wvv/zSyhjUdxEQAREQAREQAREQgQQE8AYgDut+++1nBg8enOCM+mRhTKsQEPVhrVpEIEsEpGTK0tWoQ1vmn39+g4IJV7oLLrjAvPvuu2aFFVZw1kkEBnz88cfNQgst5JRLSZrz4osvOmUTSqmkctlll7mVLvbdd18FN46BJh/2GCgNkkSgzZVWWskQw2zhhRc2WOl98803Zbee+Gc9e/YMP1FLP5Zvx0Jxvvnmc3WVM5hMq335HRo1apRZZJFFzHnnnZd/qMP+Aw88YNZcc82CH2812eHEhAnltCWuyIsvvthxZ+XOSuXll192ynxW/Jxrrrnc9muvvVZ2ca+//rqZY445DG7FjSLE7Tv11FMNExDcywYOHGh+//33RM3nd3/UUUeZZZdd1v0Nrbbaaubpp58Oz91rr73CvwncvyWtTeCff/4xfIIgCEGwHd0PD8RsJM0Xc6r5888/45JTS+Nv4d9//01UXtJ80cLK7XsldUTrq3a7nPrz88b9TqptT9bOxy1up512MgcccIALtJ2l9vEcYIL7119/zVKz1BYREIEaE5CSqcaAs1r8+OOPb3bddVfDyxAxm3iRIS4SL8hvvfVW2c3G4imJookXOFa64MXjrLPOKrueVjhhzjnnNNNOO61mfhrsYvP7xx31q6++cr9xlEBY7KF8QalbjvAC8Omnn7q/y/3339/06dPHnU4aL+BPPfWUG1CyuiTK2n322adk8Wm2L1rZX3/95fqN8ivJi9cff/zhGMEp+kGpc//995s333wzWnxZ2+W2Jb/w999/380Cw3nMmDH5hxPtP/bYY07B8swzzxhW+uTDNtewHKtPWOKe/Mknn5jffvstUd2dnWns2LFm7bXXdrPo6667rkFJxD1/lVVWcasLFWsfL9VMeDAJwuqo2223nfst9O/fP3TthiV/Dzy/arEIRbH26Vi2CHzwwQfudzDzzDO7MQytO/LII909c4YZZnAhAQq1GEX36quvbmaccUb3e33kkUcKZY1NP/nkk909L/agTVx00UWdpTjW4v6zwAILFMreIf3bb781vXr1SnQvxLp8yimnTDxuq6Tvu+++u9lkk006tLMeCSgmtthiC8M1ZZx67733Fqz24YcfNvPOO6/Lu95665nhw4c7RR2/kckmm6xpJzWZ3GFC65BDDjGnnXZaQT6ddUATp51FXvWKQCcTsC8zEhFwBOyMO9OBVX3sLHRBmtYtzpV90EEHFcyjA20E7AA4sAMr4agzAX7/xX7DhZpjrZWC8cYbL1h++eVzstx5553uN29fjHPSS+1YBUPsedZ6I7AD7cAuBxwWMWjQoMC6tgZWQRKm5W+k3T5f/nPPPRdYZZqrH3bWtdYfKvvbmvgHk0wySWAtfso+lxOqbYtVcgR2MOzaQF/sC0pF7ejdu3cw6aSTBtbdNTzfvjQGNiZeYGNThGmlNmxMjbAtVlFTKnsmjp944onut2DjAIbtsZMY7rdsLWXDtLgNjsPdrooUHv7xxx8dS1hEpV+/foF9aY8m1XzbWvm69vEt6XwC/H3aJdHDhthVagOr3AmsUjawFoCBVdIEd9xxR3jcb3BvtQqLwCorAmvxEth4lYFVNgX8/ZcSzj300EPdvd4qMQpm515sLTfcZ9iwYa5d3N+SyEsvvRRYi3P3Wyt1L7RK3WDppZcOunbtGljlfMniK+n7PffcE1ilbrDhhhuWLL8WGayCK7BhHdz99Lbbbgsmn3zygOdZvpDGMf4+7QRBcPjhhwcbbbRRmI32W6vfcL9ZNugT9036m2WxE6dVjQ+y3De1TQREIJ6ALJns3VlinIktM4PVChZNcTJkyBDDbNhhhx1mTjnllLgsSosQUKDECIwG2CSG1n/+8x/nBhptLlYd3bt3D2OaceyVV14xt9xyS9kxt5iVfeihh1wss4kmmiisBotE4qfhhlpIymlfoTLy05999llnVYX1ULXuXLhE4fZ30kknOcuv/LpK7afRFqtcNCNHjnSWMqXqK3QciyP7Iudm3u0LT5gNy0SserASzXflCDNFNh588EHneojFRCMJ93mrADKsKOQF61isC3DHLibMwPP3gtWClx49ehirKHCulT5N3yIQR+Dyyy93MR57WjdjXJW5Lw4dOrRDVix/tt56a/eb4r5JvErcqbiHlBKsRvkbti/0RbPiJktoAj7EsMQqkntbKeH+w9+AnYhz1syl8h9//PFmiSWWcJZMpfJyvNy+Y1GF5SBBpDtLCGTNvZn76QYbbODuL1ah2KE53F+sUslZQ3br1s2NNWl7MwveAFgxY8HHbyHLwphWcZmyfIXUNhFIn4CUTOkzbbgSceHARSEtyXebO/vss90gxVpcmBNOOCGtapq6HMyLUSrYmfym7mezdA43n88//9wMGDAgp0t2FtsQI4gXHy+YtuN68Nlnn/mkRN8op5C+ffvm5Ee5hTsArl6FpJz2FSojP93OnrtFAXjpym9Tft5i+8Tr2X777Z2LyR577FEsa8Fj1bYFJRdxhFDUTTPNNAXrKXUAlwyU9fkKFV4yedkkVpe1eCtazA8//OB44FJsLeOK5s3SQdzXcH/kpTdfSCv2+/z555/d/Q4XJoSXYWtBYXChxNUOBZ1EBIoR4LdC/DMvuJuRli8ofqIuRfzt45KFe3MpYVXeu+++27ljlcrLca+kQVHC6r6lZOqpp3a/fVxFSwnxMFk1OInyypdVbt8JbXDEEUcY3LI7Q6x1ksGNlhWRvRS6rqyiTD6ULtxvUDAluaa+3Eb75jdMH4899tjEMVQ7s4+MaV999dXObILqFgERqDOB4qPdOjdG1XUOAWaJSr34lNOyqDUTyitWukDxxMNQkoyAfNiTccp6Ln7/zJazkqKXjTfe2L0YTD/99D4p0TcvLEicEmSqqaYyX3/9daJyopni2hc9XmybwM78bVsXt2LZSh5jNhalAvcHlEWVSDVtYRVHVtkkaCoByWsh11xzjXvhpJ5SQjuwfOKFtpGk1O8TRZJ184ztEhYcCNYKWELxIrnOOuu4l0YsUnjRlIhAMQL8hrAa9TLFFFMYFLbF5OOPP3Zxz3hhT3I/jrv3FiufmHwspLL44osXyxYew0IVRXUpwWKSVcRQXkWtWkudFz1equ8sNoFiLPrsip5fj22uKfGmeIZ6KXRdUXJjDcukC9byH374obGujf60pvrGwhVrNyZtmbxtBGFMy0TDTz/91AjNVRtFQARSICAlUwoQG70IbvpJXDjK6SfWUVgGsNIFL0sosiTJCTBzSFBSmRcnZ5a1nAwAr776asOKWLw4eyF4MQE6USSUIz74My5E+cLLlX/Jzz9WaL9Q+wrlr0U69x2WXSagq7diqUU9xcrcc889XQBhG0+qWLaKj7GwAkFZGWSXeunB5ee+++4zKKUmmGCCiuvsjBNL/T5pU6EA+F7JhBUCL7YEJyZYuo2jYnjZLcdaozP6rjo7nwD3Rf8bpDVsEyy6kGAZuswyy7iVOiu1oCxUNuk2QoW7t9WibKwcUawTAB9rWQLuo1RJ+gJfqu+UhQsW90bKR4GDMr6YNWIxFpUey7+mlFPouto4eI7JwQcf7FY8xoIetzrcuZtJeG7zHEHRRPiJRhFNnDbKlVI7RSA9At3SK0olNSqBSlaTK9VXFEusssTLAS/UkvIJ8FCWeXH53Dr7DBQnvAQwyGV1sDPPPDOVJmGthLA6W77gcpZUKVGr9uW3Kck+qwGxkhuWBGlaUyapmzw33nijsYGmnUKDl5S05YYbbnAulMRoYeU8XgoLCdZcKFmIrcHqao0mpX6f9KfQb9RbKhAbhxhM3jpuqaWWcqtzoQDkpdfnazQ2am/tCcwyyyzuXsJvBuG+0jPiphxtwZNPPuni95xzzjlmyy23jB5KbdsG8Dbff/+9qye1QscV9MUXXxgskexiBy4FxQvPHCb2sJQtJkn6jssvf89YESIor3ju7LLLLoYJxHoJE23cE7DS9QpDrisrTuYLrsjk9TLddNMZrCdhk8RV0Z+X5W+sfZmw5XnJBG4jiZ84ZUyLC7REBESg+Ql0qiUTDys+uFLxYVtSXwK1YM7LIgomBjsrr7xyxUuB15dE9mpT8O/sXZNSLSL2DoGLUTAx6CfwbKUuYPl14QaAxMXpIi3qKpJ/rt+vZft8HeV8E7vIrhDlXD/KOS+NvLyQYGFEHA8UGyjD+RBzBUGxUU3gbX4DvMDa1Z8McV9KWa4RXwOLBCzSfFtw/0DsqkouLe7auwwZ+K/U75Mm4voSJ8QUQwjc6xVM7PMsIY0XRWKeSUSgEAEU+sS7w1oOyzgsJIl9hxDbB8s4hLhhLG+Pggn3WGLm8eHeiKC4IU+58vzzzzuLH3/eG2+84Vzl8hXLlO2VQz5vku9oH+zKeObdd98NP9z77UqmoYKpUB9K9d33YY011gjLph4CnaPY8ePFYn1gEoPxHwoeL8SPQjHmhYUoKNdL/r5PxxWQv3+7aqVzmaUc7qW0DyEOk13Fz23jisx9kvsEVmRYQC6yyCKGOFfNICjZUTDhXt5oCibPX8G/PQl9i0BrEKi7komHFGa+zEiuuOKK7oP5J3F82CedD0onSWMS4PrxMnHXXXe5AIzMXtulhd2MPi9edonZxGbdjUkgnVZjycRsZbluUOnUrlLKJUC8GV5ebr31VnPuuec6JQV/C2mJfxHHlSEqzDDjzkDsj2JS6/YVqzvuGO3hHgEzbwUTl69WabwMsQoRL0O4qfkPLzIIKwBi6VSJ8DwjXhXxTHD9KqRciZaNgomYRMRx8W1B+YXwgkwaAYqzKrzoEk8m//dJe0kjKDMKxTjht83fin/Rj+ZBwcR1KjceTrQMbTc/ARYPwNoFq0FiILGqIe6WyCOPPBIqnLg38zePUgp3LP8hiDbCJAF/s+UKSg6voOZclCi9e/fuUAxlR1dQ7JChQEK0DwWyhMmF+lCq7/l9CAvM2yjWB/6GmVz0yh9O3WGHHVyQcl8MFu7RVYbZLqTQJ533hp7WKo2JS+IIYrWEsO1XVSPYN25kKJZwv+beffPNN/sqG/obJR/eAShGWeGwUUXW+Y165dRuEaiQgNX4102s/3Bgm5n4Y5e8r1vbWrmicq5J0rxWkeSQvvnmm4GNSxPYWfrADvoC+zIZXn8b3DWwg4bAPjwDOzgL7GpbrXwZOvTdzvw5VjZGS4djSqgNAX7fdrawosI33XTTwFotBVbJVNH50ZP+/PNPd+35u/FilSKBfYFyf0c+jW/r7uXyWnesaHKH7TTbl1/4J5984tpgrX/yDxXcf+2119w51vS/YJ5KDlTSlmg9VhHu2mVXd4wmJ94+//zz3flWyZT4nEIZX3/9dVfWxRdfXChLptLtalSBVZQF1n0zbJe1LAisRVJgY6WEaXEb6667bmCVBIF1MQoP24DfgVVeBX369AnT2LAxzoIFFlggJ63WOzzTuD/4Z1ut61P5xQnw92mVlh0yWdeugPtnpXLVVVcFDz30UKWnlzyP+7hVPpfMV02GRuuDdYELeD8oJtaSK4BdKbFKroC8+WIVjoG1DM1Pzvw+903uO9ZKL/NtLdVAa23n+mInTktl1XEREIEmIFA3SyY7SHQmr/ZmmVjsYN3NbjKLIakdgVrG/6BsZvOZcSIALqu9EHsEaw9cSVg1CJcZVhJidgoffEyhcRchngnm0K0qM800kyHOhIJ/Z/8XYF9KXJBRVsUi5gAzj9EPbhxesNJkRi/O4sPnifvG0gMzeVwRcK0iCCvuAQRn3WyzzXICZxO7gTqIrYGU0z7azblW6RvXjLLT8tviC8BFAomb7fd56tUWX1+x7yRtweqQYKzEdyIOSPQ34Ld9IFpcXuB8/fXXF6s29hjXnXO99UVspnGJBFSPC6peKH3VVVftkD9pfbiIEh/MKozcfQu3HbaxQsDdw0tc31kliecDzwKsNvh9sIQ6lkzRFUt9GfoWAaz/GCcSp8cLFnXVxODhNxhdqMGXm9Y3lpJY9tRSGq0PuMDhulxMiLGUxDIYq0fyerHvae43glV4ownPe6y8LrzwQmMn3Rut+R3ayzMLUazRDmiUIAJNSaBbPXrFS5C1aKm4KtzoeFBIakMAdxVMm3EfSUtWWGGFgkVhyszHm7KTkZczO2vvPsQywOzcm1PjgoEJNJ+FF1443GYw0eyiuEyNcYW9ogDFD/Ej8gV3Ub/KEC9EKA7tbHt+tpL7LG9PEFYUVcRm4IWKl3KUuFHxdfgg4eW0z1oCVdy+aBv8dn5bfLpXMlmLFJ/U4btebelQcUxCkrYQ3Jv4Lkgh9w9WP+IlmPhK/A4qcYcloDDnFlqtLdr8Qs/eYun5QdiT1jfvvPO6AOfbbbedU4IRjwbXGV6SorGW4vqOixNBiQcMGBAG9kVpa60ywhgs0X5pu7UJEGsHhS5/P0xWpSW4udZSCBdRa2m0PuC6WEvhN8I4t2/fvrWsJtWyuYaElyC2WK2Vkqk2vEhhfuIUJVPcxEeRU3VIBESgAQl0wRqrlu3mZajaWUgGvAxAfayMWra3FctO4xpFuaFgsi4F0aSKtnlB9oon/40CigElM1pRhZPf5qW7mQQ/fAYZ0ZnaZupf1vrC74rgmvxNdKZg7ULQUyyW8hVItIsZfCwCUdbmB5ZNo93ENWEgSIygzha1Jf4KYOWDop7gxfWQcutDKUe8m0ruybwUooT1sVfy+8czBsUXy6vXS7CoZsKLZxv1S0RABESgFgT23ntvM2TIEHPFFVc4pXst6uisMtdff323GAreDBIREIHmJlBzU5BqFUzgx8KG5WAZ5Glwl/4PkhdqAsqmtdR6GgomesnMNysz8YkKM/Be6cQ3q6pg3YEQXNYrnLzlkw+YHC2jUbYxL8bNhMDOfvWmRmm72lk7AljxzTPPPDWpgICp/M1kQcGktsRfYpQrBANnKet6SCX1sWpfpRJ1d6m0DJ0nAiIgAo1GAItnXEBtLFMXaqLR2l+qvYxpL7vsslLZdFwERKAJCIxXyz6kbQ3QqMt21pJxWmXHWUtUUnZaCqZidbOKFqugoBQjPg2uF8S3YWZk8803N1iBsHzt2muv7WIaoZxhtp+VR3hpxaWpUUQ+7I1ypWrTTlY4Y+nom266qTYVxJRKHLDoKkkxWeqWpLbEoybm0z333OMsheJzpJta7/oKtZ6VaPl7wKJVIgIiIALNRGCXXXZxCqZrr722KRVMXCvGtFjmf/XVV8106dQXERCBGAI1tWTCwiRNUbC4NGl2LAs3oWoszzi/syzN5pxzTsMnGufJrjASxnjyFk8+TgpWGt7Sycd74tuuDtYRTCemTD/99AaLAOKvEHtH0hoE+B0Sy84L8UfqJfmWg/WqN64etSWOinH3hPgjtUmtxiopzRYR04O4T3xYJEIiAiIgAs1AAHdkLHyYCLUrwTZDl2L7EJ04ZSJYIgIi0LwEahqTKclKEOWiVTyEcomVlx+XRPzB33rrrbJOzEIcmyQNZuUj72rHbLjftsveGmJ/5bvaoXiacsopkxRdszwbb7yxi0N111131awOFdxGICsxmXQ9REAEskVAMZmydT3UGhFoFgLbb7+9ufLKKw2WyxtttFGzdKtgP4glOcAu8BBdcbRgZh0QARFoWAI1s2RiQCZpPAJYIhHzCFfHJFZNLKsaXZ496z3G7WOZZZZxn2hbowonFE933HFHTpynqLUT2/WMj8TMz7nnnhttrrZFQAREQAREQAREQAQamAChH4YOHWpuv/12Q1DsVhCtmtwKV1l9FAFjaqZkqgVcVlFCedVZLlm16FNWy0TJ5Dnfe++9Lv7He++9Z1DSMOuC+w7HfZ6s9iNpu7Bg4sPS216I8xRVPl1wwQXmiy++cIdRMkWtnlA8sQpWLQQlE4G/P/vss4KrLdWiXpUpAiIgAiIgAiIgAiKQPoEtt9zSXH/99S4GYiu5jjGmbaTJ6fSvvEoUgdYg0FBKJlyaJPUj4BVIfI8aNcr88ccfTtFBnKAjjjiifg3ppJp69epl+ETNl7/++uucOE/MPp100kmuhVNMMUVsnCfc8KoR78NOXKZCS3pXU77OFQEREAEREAEREAERqA+BzTbbzC1Wc99995k11lijPpVmpBbGtCNHjjSff/65W5wnI81SM0RABFImUN3bb5HGeAVFkSwVHapVuRU1pkVOwnrn0ksvNQcddJA55ZRTzKBBg5xpb4t0P6ebM8wwg1l99dXNwQcf7Gag3n33XfPrr7+ap59+2rDyEdZMzz//vGGVEB6kWN/xvcMOO5ghQ4aYp556yvzyyy85ZZbawWpsrrnmMoUC32Pd5z+lytLx2hJgVnK55ZZzykBmJq+44oqKKvz7779Nz549ww+rJXrBuu7AAw808803n1lppZXM4MGD/aHE3yiNsb6rdjYxS22phj3KYlYtI8A1s8s33HBDYpZxGV9++WW3CAH3A/52WZDgtddei8vaIW3//fd3Qf4J9B/9nHjiiR3yZjEBq8tTTz3VLLnkku7eN3DgQEMsvHJlq622MltssUXOaXvttVf4N7HqqqvmHNOOCIiACDQCAeJsEpKhFRVMXB8/cVpoTNsI11BtFAERKE2gppZMffr0MQy20xQpmdKkmays888/30w77bSG+EsERn777bed0qR3797uBSpZKc2ba7LJJjPLLrus+/heBkGQ42rHy/htt93mLMLIw8tnfpynYqsl5fuwo1RC0fXSSy/5KnO+UQhOPPHELrZWzgHt1IwAigoUFEsttZTZY489nOKPFWOGDx8eWrslrZzfD8v8rrjiii5OA/dShDR+a/PPP7/ZaaedXID+fffd13z88ceJlU1//fWX2Xzzzd3v888//0zapA75stSWatijOD/kkENM37593XV76KGHnHID91T+jsqVxx57zCmsUA6jKOFaXnfddYZr+Mgjj5hizzDyotCfZJJJOqye9uOPP5bblLrnHzt2rEG5ysqe/A389ttvTrn+3HPPGbhwT0oixAOEWb4LyVprreWeOSjsUWZJREAERKCRCBB36eGHH3YKppVXXrmRmp5aW/3EKdb5rRKHKjV4KkgEGomAHdTWTOxKcIFlkdpnv/32q1lbVXA8gdGjRwd2dbXArh4XZrBui4GdiQisxUaYpo1kBKzCIbj55puDww8/PLAvTMHMM88c/n3Y5bmDNddcMzjssMOCm266KSCvF2sZEEw33XQBf1OLLrpoeE6pvy9rFeGL0HcCAvCM/tYTnOKy2JfrwFoWBVZRkXOKtWAJunXrFvz000856aV2rPLHXeP862etNwK7MkvA36UXa1kYWOVv8P777/ukgt/2Zd+1k/z09fTTTy+Yt9SBrLSlGvYjRowIJpxwwmDBBRcMuK95OeusswLr5hpYpZBPSvxtle+BjV0XWIvF8Jxvv/3W1WOVxWFa3IaNA+eui7Vcizuc+TRrbeV+izZ+X9jWRx991PXpkksuCdMKbVir2cAqllx+fqNsx0m/fv2CBRZYIO5QzdL8eIZviQiIgAiUS4D72eSTTx7YScJyT226/NZd0I13m65j6pAIiEBIoGbucvYFxs3YYu2Slqy77rppFaVyEhIg2DXuXVgxebEvzeayyy5zVmq77babT9Z3AgK4zmAqffzxx7tg6vikMyOP2fSee+5p7ADE+elvuummztqpe/fu7u8IFzz7ouosW5K63dCcM88801mfEchdUjsCWG7gRnncccflVGKVhgbrDqz/vLzyyituqeJy3SaxiMLKZptttjETTTSRL87suuuu7hrzN1lMnn32WWcFhSUTrmXVSJbaUg77/D7DBB5bb7214b7mBYs0rDbLdXfEcgfrMNy8+Fv2giXoaqut5izP/v33X5/c4dv/bXt3gg4ZMp6AhZFVAJl55pknbCkunfPOO6+xSqYwLW6D64CFnlXiOO6KPxdHSWkiIAKNRuCff/4xjAWeeeYZN9bjHtnqwjNO7nKt/itQ/5udQPuoukY9ZdCJy0e1cuSRRxZ1M6i2fJ0fTwBXORRM1oomJwMrq/FSizuInVF2CpKcDNpJTAA3OT7R4I/EeXr99dfDIOP33HNP4vLiMuJ+soIN4M5Hkj4BVhtEoRcVlAnEXWBFxsUXXzw8RBykK6+80ikcylHCo5xCcOuKCnVbizhjLZmiyR22u3btaqzlkosXhsKyGslSW8phn99nv1pk/t8F9zvK9f3MP6/QPq6zH3zwQYfDLFrxwgsvuFhdxRYC4G9+ggkmcKt3ouDHRW6hhRZyLyhcvywLyvKvvvrKKUHz27nEEkuYO++8Mz85Z99OfbmVS3FRhL0U4zl4tCMCIlAjAoQfQPh+8skn3bZXBFV7H0J5zgQ5rmFMJuJOL2mLy8QE0SeffOJiIYqJCIhA8xGouZKJwbt1PzG85FYi1pXB8LJGzBFezPBhnmWWWSopSueUSWDo0KHGum8UfDlgth8LDYKxomhKQ5lYZhObNjtWEASQ5oNgVVGtEFCYgY6ktgRQXGCpxoDSurWZe++913Af84IlG5YerNJYjnjF0DTTTNPhtKmmmsqw8mExIRAznzQkS22J9qcU+2hetq3roUviGqEI8fLWW2+5lW/ylev+eLnf11xzjbNEJE5RMUHJxN86Fj0sGoBlFM8/rhtxp1h4IKtS6jfx888/u7+HQnGZsM47++yzs9o9tUsERKDJCKBU4t2EbwTLcRbFQLgXs+0n6LCwLFdYkZnFGxgn5z9jyi2r2fJ7a12smVhwQyICItB8BMarR5eYCUDRVImwhDxWTLw8bL/99m4mmIDJrFjDSxyzBJLaEMCKCbctXngKCSuqMUvD6mn+JaNQXqVXRgAlXhqCK061s3JptKPZy3jnnXfcAPX77793XWWmDisNLwQvJtA0LlTlCAoHpEePHh1OY3Bcz7+/LLUlCqMU+2hetnHlIggpVpk2dpA7TMBvPymCkrBaoVzcihlUH3rooUWLY7VKlDAolGwcLzfLy28FKyhWrMyylPpN0Pbvvvsuy11Q20RABFqEAGMhJka9goluewVT/jZ5UP6XM37CGh0XORufzr2rRCcxWgRx0W7aWK9uhVxNfBbFpIMi0NAE6qJkghA356QzAX6mE8XUsGHDzBFHHOHO5aaN+8nyyy9vHnjgAcPLGqvwsJQxKwSV69rQ0Feuxo2HLy820VhMharkBQ0XEBRNknQJ8Hdz7rnnplaof3lOrUAV1IEA9yP+dnAdwtoP5fiFF17YIV+5CVgrIcyO5gtLxONmVS/JUluifS6XPf1gNTcUOqussoqZbbbZnHUT++utt56ZYooposWXvX3DDTe45xSrSd5///3OOqlYIViOMqFC/CbuqcQlOumkk5zl21133VVXRWKxdsYdK/Wb4Jx6/kbj2qg0ERABEUC5VMlYiHMYk5USlFUomPDAYDKc1YElHQnkr5rcMYdSREAEGplA3ZRMQMJ1jhl9lEfRGCVRgHblLBdzBIVU/s2ceBcM/M855xxnfsoN3K7C42aizzjjDLdENO4ExAm6/PLL3XLf0bK1nZwAVkzM8nu/9GJn4r6Doom4Qcy6S7JNIDpzl+2WNnbrmKljUIrSHOV4tUKcGiRuKXvSsGaql2SpLXF9Loc9Syi/8cYbxq6MZuxqgMau/uheDGBarrVZtC24fqFkXHrppc3TTz+dqCxmy4nhlS9YlCKl4m7ln1fP/VK/CdrCdZGIgAiIQGcR4L2imjFQKUUTFswomEaOHOmeI8QvlcQTUPDveC5KFYFmIVBXJZOHxk3+5ZdfdgonlE4olPiwjekk2yikSgl+vDvuuKNbKQlXEawH9t57bxebBKuanj17uqCp++23n1OApOH6UKpNzXAci7C7777buXgk7Q/KKAIaY1FG/BFJOgSwapBknwCuiChYsUSJCoofgrrHBYOO5kuyTXBvJL8OLJsIukyA6HpJltpSLXvcvIjzhyvbWWedZTbYYAP3LHrzzTdz4jSVw5YXEZ47rASIVWgS5QovJ4MHDw7d9qL1MaGClBvHK1pGrbf5rTMRlP/7pF7SWFnTWynXui0qXwREQATyCaBc4t5crVBGnKKKQNYomHALJgZTOQt7VNumRjwfJdMPP/yQyvioEfuvNotAsxPoFCVTPlQUSkmUSvnn5e+z6tJhhx3mBum4j+BeQKBwYmIQfA/XOvaZsX7xxRfzT9f+OAKsarTgggsaAhSXI7jW8UHB55fiLud85e1IAAuxtIW/EUm6BH755RenYOXeEhVWqkFBUMhyM5q31DYDMtyurr/++pysrNqFAh0rnHpJltpSDXsGuFi/Hn744TnorrvuOoN79rbbbpuTnmSH+ycTKSiZrrrqqsQuYihgWFltp512Mix57YVryzVm0mTOOef0yZn83nzzzd1zN+rSiaIcS656/j4zCUeNEgER6FQCaSiYfAfyy2KiBwUTzyNc5Oabbz6fVd8FCDCOQAj+LREBEWhCAtZ6qCVkxIgRgX1hD6z7QmBng4nCG1hXiMAOioNLLrkksC+CLcGhVCdtkGLHxr4olcpa8Li1agqs22Ngg7IXzKMDyQjwO03706dPn2SVt2AuWFt33rJ7Pnbs2MAqygO7IlhgB5+Bdb8KrAtvYBUDgXV/CoYPHx6WSfk2FkFgrZvCtPyNP//80113uyJgzqGLL77YpVsFRmADiga33nprYGPhBJtttllOPtpAHdF6oxn83/npp58eTXbbVgnpzv300087HIsmZKUt5bCP43LAAQcE1gInsLGZHFPrKhzYlc4Cq3iKdjdIwsXOZAfWosddc2vZ5s7hvOiHa4tY92LH2Sq0wnpoC79BnlPPPvts8NJLLwU29mBg4zO5a+0zct25vrfddptPKvht4zsFfPKlUHr//v075E9anw1c7tpqJ3MC++Lg+sCzwK7iF9iJn7AJcX0PD47bsPGxgrXXXjs/2e3zjLErmsYeq1WitbB214ZviQiIQGMR8H+/aY6n/L3ALhYRWLe4wC6SE3z00UeNBaaTW2sntIMDDzywk1uh6kVABGpBoJu94baEEND1v//9r/vQYdz1sHB67LHH3Ko9LBNtB63O0glrJ+IRYfrfasIsPO49u+66a8Vdx/oGqzIsmoYOHVpxOTqxNgSWWmqp2hTcwqV27drVxfLZZ599XMw54s4hWDDdcsstzlXI47HKG+cWbJUNPinxN1YuBKXGUgbXLlyUsNIkJl1UfB1Ri5Lo8WLbrIaH23Kp9mWlLZWwj3LBeojlqnG9RqxCxOy5556OcZRTEi4E9/YrFJ188snR08Ptgw8+2Ew44YQuthaco6sCEuSbY1xPrKkQnl3MjBMM3AuudZybZLU23P7ipFg6QcejkrS+eeed1wU432677dxqelbp6p6pBL7HktgL8a7y++6P6VsEREAE0iYQ596WRh08F3gG8xziPs39WpKcgIJ/J2elnCLQcARqoblqtDKtO0Jg/acDrAaYjbAX0X2wTDjuuOOC5557rtG6VFF7bWwSN6N/7LHHVnR+9CQb6NgxtC9N0WRtl0lgkUUWCX+P/ndZ7feRRx5ZZitaJztsK7FkihKy5vKBVVoE9sU8mlzWdiFLJl/ImDFjnNXN33//7ZNS/bbxcwL6kUSy1JZq2GOFVMjyy3Moh4s/p5JvritWQXaFwoKnWyW+e24VzJDygXLrwzrYKkRTbkVbcbJkqglWFSoCTUuA53q1Y6f88+1kamAV685y37oFNy27WnbMLuQU2JiFtaxCZYuACHQSgZaxZLIPh4JiXSOcLzX+1Mjnn3/urJywdGKVtUGDBrkV7LBw8p+sx8bwnfWzN3xjtYWFFoIlRL7QV6wXiKtUrbAK4AknnOCC6WIhxkyPpHwC1lXGWVmUf2bhM1i+V1I7ApNPPrmp9Yoy3bp1c8va16IXN954oyGwN/1IIllqSzXsp5tuOsOnkJTLpVA5SdKxAMIqqJAMGzbMPPPMM+a0004rlCXV9ErqY2EOiQiIgAhkgQBx7dIWYglxn2M165lmmint4luiPOIy/fzzz8ZOqiiOVUtccXWylQh0QbnVSh2upK88SFDQoHTiY2OAuJuhd6vje4oppqik6JqdgxIpPzBhXGV2didUOOEmgjKIFY7Skq222so8/PDDLtC6XjrKp4pyMG2lkP7kC18HlpCP/k0UzlnbIzaemUH5Peuss7pg37im+WXsa1uzMdZy0ynJbCypWldVsny1JR4RLhoEB+/Vq1d8hpRT611foeYzccE9kRVQealD+VUv8fdiG4fFWCvnelWrekRABFIgwLO9lvKf//zHWMtz9+zkmw8LdUiKE/BjHVal5n1BIgIi0DwEZMmU4Fqiaedjg9MZ6xoSKptQOJ177rmuhOWXXz60clpmmWUSlFqbLAyErdtf4tXdUEQRHwO/aF4k0rBiivaM+ExLLrmki8+Eok5SHgFeZmzg3MTXs1TpWOVJsk+A+A7EBfIy9dRT+82afy+99NI1ryNpBWpLPKl6K+zrXV98r41TLGHhxYfYgRIREAERSEKAsRTj4zSFMlntlZh+fOyCH+b22283xNZDmHz2Ciesm/12fsy7NNvUaGURg5AxLpP5UjI12tVTe0WgOAFZMhXnU/LoyJEjndLJWzqxXHOPHj1ChRNWTjaOR8ly0siQ1HqpUF209ZFHHil0uOJ0Hh4omggmPmTIkIrLadUT/Qx6Gv2XFVNxilmxZCreSh0VARGoNwF/H5YlU73Jqz4RqJ4A42Os9P3CDNWWyKIbLPRBufli45uGiqeoAgovCCaQogonv501b4j8PtVyn0U37Gq75sknn6xlNSpbBESgzgRkyVQlcEz2t912W/ehKB4o3q2OB5AN4OpipxALCSUOHx5OtZAk7nHF6h0xYkSxwxUfwwrMLg9uBgwY4Fbwq2bluoob0cAnMluGC1e115eXI4kIiIAIiIAIiIAItBIBxlHVjqGivIopq1iZetlll3Wf6Dle4eS/b731VhePiDzzzDNPB+UTLnitILwj3HDDDa3QVfVRBFqKgJRMKV9ubw47cOBAFzPDK5ywdLrgggtcbTx8vMJpueWWS6UFUdeaSgv86KOPzCGHHGIKLb1dabmcx5LWb7/9ttltt92coimtflfTpkY618+WVTpIQknFIEsiAiIgAiIgAiIgAiJQHQE/Lktain8/iObHggc3O694Ou+88wweEgiT2JzjrZ3YrpdnRLSNtd5GyfT777+bt956yyy44IK1rk7li4AI1ImA3OXqBJpq7HLUOQHEP/vsM2OX7gwVTlg7FVtRqFBTedBVqnyIK7OW7gCslsZD9cUXX3RuhXH1K60wgXKvNYOTs88+WwqmwkhzjshdLgeHdkRABMYRkLucfgoi0NgEyh0/FettLRcI4V3BK528Aur99993zeGdwSurosqnWgc2L8ai2mMsYjHBBBO4VfqYkJaIgAg0BwEpmTrxOr755puhax2WTqNHj3arBXkrJ76nmmqqki3s27eveemll0rmS5KBBxWBy59++ukk2cvO8/XXX7v4TAT6I0CipDICpQZL8F133XUN+STJCUjJlJyVcopAKxGQkqmVrrb62qwEWK0XBU4xd7dSfa+lgqlQ3b/++muoeIoqoFDQdOvWLcfaySufJp988kLFZS69T58+ZqmlljLnnHNO5tqmBomACFRGQEqmyrilfhYBmXGt8wHEvdKI1ZW80qlfv36x9aY9gzHffPOZd955J7auNBLp5yqrrGIOP/xwc/zxx6dRZEuXwcsPH+8O579bGkqFnZeSqUJwOk0EmpyAlExNfoHVvZYg4P+OK+ks8VRRTmVlAZV///03x9XOK59++eUX1z08I7zCybvdZXVVTmK14i737LPPVnJpdI4IiEAGCSgmU0YuCi+3KF74IN98802ocBo6dKg57rjjDLMSKJx8EPH555/fKRfS7sK7776bdpE55dEHVvkgMHrv3r3N5ptvnnNcO+URQKkkxVJ5zJRbBERABERABESgtQgwVkJJhJV3uWEmUNRgxZQVGW+88QxW63yiMnz48BzlE6s6f/nlly7LzDPP3CHOU69evaKnd8o2cZmuvPLKTqlblYqACNSGgJRMteFadanTTz+92WKLLdyHwoYNGxZaOh166KEuSN4cc8xRs5XqopYxVXcmpoC9997b9WmHHXZwgcAV7C8GkpJEQAREQAREQAREQARSJeBDCZRSNHnrpc5wkau0w3PP/f/tnQWYJNXVhi8srgm+C4uzOAQLLK6Lu21wWyS4BLeFLBLcJXhIcFk8aHDdJbgHCe4Oi9V/3vNzK9U1LdUz3bPdM995npkuuXXvrbeqq6u+OufcAYG/DTfcMK0CkSl6OpHn6aqrrgrDhg3z9exjuTxP6cbdMIHINGrUqPDkk092EM26oXk1IQIi0AQCCpdrAtTuqJKwOv4uvvji8Omnn4Zvv/22oc121w8q+Z9+/PFHTwTe6LC/hgJRZb2CgMLlesVh1k6KQN0EYphNMwfGqLtT2kAERKDLBKLgNHz4cBdiqBDRBfGFNBV4P/HX04ywuig8xU8EKMLwxh577A6hdjCZaKKJmoZh3HHHDWeeeWbg5bNMBESg/QlIZGrzYxhvfBu9G911I82IGSQuX3fddcOFF17Y6N1QfSJQFwGJTHXhUmER6DUE4m9td/029hqw2lEREIGWIUAi8TiiXRSe+CTxOEbO1nyep2mmmaYh/SfxN6F/CE0yERCB9iegcLk2P4bNervSrHrzuGefffZw/vnnhw022MDD5vbZZ598Ec2LgAiIgAiIgAiIgAiIgAg0kUCfPn3Cggsu6H/ZZnghjNgUBSjyqr733ntepH///h3yPM0yyyzZzQtNEzL3+OOPVy3b7FQeVRvXShEQgboISGSqC1drFp577rnDc88917DO4RLbnbb++ut7AsY//elPngh8lVVW6c7m1ZYIiIAIiIAIiIAIiIAIiEAZArwQ5m/jjTdO177zzjtpuB3i0xVXXJGOGP3b3/62Q54nPKCqGSLT2Wef7eF6JDVHUOLv3nvv9VH9ELmi8SI8jrgdwx3jOn2KgAi0BgGJTK1xHLrUiymnnDIQy0zSvEbY2muv3Yhq6qrj0EMPdaFsm2228fxMvBmRiYAIiIAIiIAIiIAIiIAItBaBaaedNvC3+uqrpx374osvUuEJUYjw4pNPPtlH9BtnnHE6hNrxUnvCCSf07RGZCNcbOXJkuOmmm/zlc0y8njbw60QUoOJyCU2RhD5FoHUIKCdT6xyLLvWkkUmzGd51dNjXX38dFltssdCvX79w++23j44uqM1eTkA5mXr5CaDdF4EKBHioWW655fyhqbvCySt0RYtFQAREoG0I/PTTT2mYHcJT/OOeH5trrrlS8enggw8OM8wwQ3j11Vfr3r/uGrCo7o5pAxHopQTkydRDDjxvC7gB7qpxkR5dxqgV5GdCaNpzzz3DSSedNLq6onZFQAREQAREQAREQAREQAS6QGCsscYKeCnxl7UXX3yxJM8T9/yMNv3uu+9mixWeHjp0aOAFgF4CFEamgiLQVAJjNrV2Vd5tBLioLr300l1qrxXeAjDS3HnnnefutXzKREAEREAEREAEREAEREAEeg6BOeaYIwwePDgcffTR4dZbbw077LCD79y3337b6Z1sxMv2TjeuDUVABEoISGQqwdHeMyTHI7dRZ6wVBKbY72233dY9mYYMGRIeeeSRuFifIiACItAlAtdee62/TeUza3H5ddddl10crr76ai+fX15SSDMiIAIiIAIiIAKdJkBOJTyRGmHNEpp++eUXzxnFZ9by89l1+ekffvjBE5vnl1ebJ4VJ0TQm9fQl22bR7YqWy9adnS6yfdF9zdbbjOkifaXdfDn6T24x/nq7SWTqYWcAF+l6Qt4Y7YFQu1ZLmnfiiSeGQYMGBRKBf/XVVz3sKGl3REAERgeBjz/+2JOK8pm1uPyjjz7KLg6VlpcU0owIiIAIiIAIiECnCTRKYKID+aTgne5UbsPNN988MGpeTHT+9NNPhxVWWCFMNtlkgSiM1157LbdF6eyHH34YZp111sB2RQ3HgZlnnjlMM8004cgjj6y4GXlsiWaZeuqpfdS9N998s2LZ7Iqi+0BCd7zOSMQ+55xzhlNOOSVbTaHpP/7xj2HDDTesWPa2224LjC7et2/fsMYaa4Q777yzYtlmrrjyyis9R9gUU0zhoylW8qzjeG+wwQaeRxgmsb/HH3+8Hy8G5er1ZoqbrIcSMLEp2WWXXRJzSSWTd/o399xzJ4svvnhi4lJL7/l///vfxEauSOxL3NL9VOd6DgG+J3xvZCIgAiKQJcDvJdeHVv/dzPZZ0yIgAiLQ6gTitTX7nNKVaRNCmnIft8kmmyTHHnus4zSPpMRe0iennXZaYiJEctBBByULLLBARdSPPfZYYgnO/TfkySefrFguu8LEDq/z9ddfTyxZemICVXL99ddni/j0N998k5gwk5jQlFiS9eSAAw5ITHDqUC6/oJ592H777ZOBAwcmb731lv/ZqIAJ/StqNlpgMvbYYyfrrbde2U2+//77xIS05Oabb07MMygx73HfJ/rYnfaf//zH+3H//fcnH3zwQWKCYrLHHnuU7YKNxJ4ccsghzvyWW25Jxh9//MSS2XtZE+8SEyTLbtebFsqTya5kPdXwTrILYHjhhRfc1dIu5P757LPPhgcffLDlk+NNN910ngickJVGvuXoqcdb+yUCIiACIiACIiACIiAC7UIAz6NG2ueffx5IH9JMGzlyZPjss8/CzjvvHExc8GiQN954I/B8lbd33nnHPXP23XffUI93ywUXXBDMUSDMOOOM7lmz4447hr/97W/56j3aw8SvsNJKK4U+ffoEcyIIL7/8cody+QX17IOJRGG33XYL/fv39z+iTC677LJ8lWXn8eDaa6+9fPuyBWwhXkGbbbZZWG211QKjPK+zzjoebsazancaXkxwXHLJJcNUU03lz57lmBMKh7fVn/70J2fOgFWMosixlv2PgESm/7Ho8VPtOOLCyiuvHE444QS/gCM2yURABERABERABERABERABHoGAcKwGmmNFq7yfUNQIvQNQQRjBL0ZZpghsDxvk08+uYsoW265ZX5V1Xnqmm222dIytFeufkLkCOUjrOuSSy4J5snk4ke6YYUJ6iq6D/369XMxJVZFW0WELMpvt9124eCDD3Y+cfv8p3l5heOOOy5dbJ5ELp4RhtadVo75J598EsxDqaQbiHns18QTTxzMiylsscUWLpANGDCgpFxvn5HI1NvPgDbYfxTwrbfeOpAQnCFPZSIgAiIgAiIgAiIgAiIgAu1NAK8jvI8abc0UmvBYyQtjk0wySUCQyNt4440XJppoovzimvP5NirVHyuysLpg4XQBzyHarGX5+ilfqQ1ELEYBfOihh8Jdd93lHlX53Jbl2jvnnHPCuOOO6yJYufXlllnIWth0001ddEJA607LM4EHVu64stxC35yFhUAGom/yYhRlerNJZOrNR7+N9h23UYY7xUVTJgIiIAIiIAIiIAIiIAIiIALlCDQzeoME4HlBgXkSdDfK8m3Uqt/y7QZGyh0xYkTYfffdwzPPPFO1K/n6KVypDV7yL7LIIp78m9C8ffbZp6pnEnW9+uqrPuI5IX+EEb777rvhyy+/DC+99BKry9oTTzwRllhiCffEIhSxuy3PBB54q1USu1hH2CCJ1klJc+GFF3Z3l1u6PYlMLX141LksAYQmRkIYMmRIdrGmRUAEREAEREAEREAEREAE2oyADbbS8B43U2Cis+Qmyo7gxjD2NlhRmNHyJzXK8m3QXrn6ifC4+OKL02bxqGFEOrxrqlm+/mr7wCjfjJpmib8DI9n9+OOPYZZZZqlWfXjllVd85D1ySTGqHALMo48+GnbYYYey2+HRxuhypEjZddddy5Zp9sI8E5gjMOU9w+Dxl7/8xTnQJ8LnyONUi3mz+99q9UtkarUjov5UJIBKf/7554fzzjuvU8NnVqxYK0RABERABERABERABERABNqewDLLLNPUfaB+G9UtXH755R4yddZZZ3m4FDmOsIcfftg9d2p14r333vMQtHLlCBk744wzwkcffeQJpXn2QazBbOSz8MADD/g0OZ9Iyh2TjpMjiNxCJADHCHGjnbzVsw+nnnpq2G+//byKL774wgeVwrsJq7QPq666qg88xeBT/NkIfJ5UO4YxZveBOmy0tkA7JP8mfJI/xCys0j6wjqThNvoek26EC5LTKdp3330X7r777mAj2PkiPrPzsRyfNoJgsJHt3AuM7RCSIvNRo0b5duSjIhcTeYJ5JsUQ36655pqWH1DLO9uN/yQydSNsNdV1AhtvvHGwISODDSkZ7rzzzq5XqBpEQAREQAREQAREQAREQAS6nUCzvY6asUNjjz22CwyEguE5dO655waiLfBowUgEfeONN9Zs+rbbbgt/+MMfypYjFy3hdySTXnjhhcPyyy8f1ltvPS/L808UPxixjpHEqQeRi7YZES0KXiynnbzVsw/kxiXMbfbZZw+LLrqot7HCCit4ldX2Id9mdj67D6effnpAvEJYI2Qt/iH4YJX2gXWIb4zcQQNPFQAAQABJREFUF+2+++7z0eniPGF69PX999/3RQhazJcT3khAvueee4bf//73nnQdweqoo47y7chBxXZ4rGEwv+KKK/z4wIUoG9jL/kdgDEtalfxvVlMi0B4E1l9//UDsLq6XjYyBbo+9Vy+bRYD4aly3Dz/88GY1UajeH374wX+4YmFGC4kuxk899VS49NJLA0PK9u3b19/+EH9fr3FTxI8nb5XKuWAXqa+zfeGN0AYbbFC2ie233z6sueaaZddVW9jZvlAnNwokqCSHADc3vAHkHKgUh1+tH/l13DThhl50uN/HH3/cE2zyVpKf53nnndeF9QUWWCBfdUvOc0PH+UluCN5C8gBxxBFHhAknnLCu/pbjhgt9vHHnxhu3/e4yvifLLbec511ox4ei7uKkdkRABESgXgL83g4dOrTezSqWb8ajLb9J888/f4mgwW874gPD3XfG6CejzjEqXCXDo2f88cf3BNqVysTlCCfct4w55v98SKibe8WVVlopFiv5rGcf6Mukk06ajqpHRUX2oaTBTszU2od6q+Q+85RTTnGu5bblHhVPpnxy93JlOf54NpHgPBr5sPAU+/TTT+OiXvn5v7OwV+6+drpdCeCiSIysEoG36xFUv6sR4EebWHDi6nmDRMJFjGXEffOmhrcmDJmLV1+9IhNvpHhbQ33RHblaf8qt60pfEFAQyXChfvvtt0v+cEGv17rSF25uBw8e7G+1GJKWIXMvuuii8Lvf/c7d1OvtS7Y8N83/+Mc/OiQIzZbJTuPCvdhii7kb/Oqrrx74wyWe44/I0er2008/hTXWWMNv3tZaa62w8sor+xveFVdc0W/Yiva/Ejd48H3gDSxilkwEREAERKD9CfA7zG9uI6wZOZ5ivwiDI0wqGmJOZwUm6uBFeQw7i3XmPxE6sgJGfn12HjEpKzCxjpHRqoUP1rMP9IWXsVkrsg/Z8p2ZrrUP9dRJMm/ONYS7SgbvIgIT208xxRQlx+exxx7zF5eV6u5Vy+1hRiYCbUnAHr7wwktslIO27L863XoEOJ/sBmW0d8xixv3ctgfqkr4MGjQoMeEpsTcs6XILH03sRz8x4ShdVm3CvKSShRZaKJlgggm8jZdffrla8YrrutIXc2/2tov2uWInfl3R2b5YToDEbrCSpZdeuqSJ4cOHl+VfUqjKjAlniYktXgfHhukiNs888yTm8ZPYCCxpcXPXTuyGJ1lwwQXTZa06YZ5xfi5aItK0izbcsXP461//mi6rNFGUm90wJ5ajr1I1TVluI8f4fvApEwEREAERaCyBeI3lPqyzf828f7OcO35/aF7Pjd1x1dajCNjLQj9PjjnmmB61X53ZGXky2ZVM1p4EGOby7LPP9hEPsiMrtOfeqNc9gQAeRg8++KDvCuFbJ554YrjhhhsC4W/RCPPkTRhDudZjJgZ5eNDmm29eMtIFI3fwZikmIKxVp92EeRJJPEI6a13tCyNwTDLJJB7z3tk+xO260pfnn38+9OvXL5D7IGt44/AWi2NYr+FmTVx/HM52+umnL1QFb9dISEnuAVyvo5FvAY8g3K9xa29lI0cBb0zJTxCNPBJzzDFHMJEpLir72VluZSvTQhEQAREQgbYiQBgy9yedNbbHI6pZRj4k6ifUSiYClQgQVs95EhOlVyrXG5ZLZOoNR7mH7iPhIzzMkKCNMBcS8PHFboewkh56SHr9bpH0j/AozkUetklIyIgZhAtFY7QQEjYyGkU9hjiFkXQxa4gkJJ4kBK6WMeIGo2UgSOHi21nral8YCYQcQ08//bQPV8uQtY888kinutOVvnBTShLHrbbaqqRtwvnIPdCZXFX2tsdFK8SvfL0ljeRmJppoIh/yNy/GEM4IG8SqvBt8rorROkv4GvkguB7njWW1zs/Ocsu3pXkREAEREIH2JMA9fGeEJrbhxY5MBESgdQhIZGqdY6GeFCSAiITnBmoxf8S/kguEh3dyebCM9fxYyUSguwgwCgW5gfCqY0hZhp3FU+bPf/6zD6dKUmmMhNdHH3103UmlqR8rJw5NNtlk6cgZXqjMPzynEMHI5cQQsV2xrvQFMQEPIYQm4uLhYyGvYeDAgZ5jrV5vna70pRIDRC+uIXiN1Wvkijv55JPdQ6rebcuVJ4k2+9jqo5bUOg6MHEMizUrWaG6V2tFyERABERCB1iXAvTv3CZXEpmyuHF4UIS7pfr91j6d61nsJSGTqvce+LfecJMeISEUMwSkfBlNkO5URgc4QYFQwjFArzj2SE2N4qGDRC4Xkxfvvv38gDKoeI5wKY/SzvHHTFR/y8+viPN5V9On444+Pizr92ZW+IMQhNpC03HL3hM8++8yFYhJFX3jhhZ40up6OdaUv5doZNmyYj/TCSGbVkmWW27bRyyyfUdhpp52C5dAKjDDYylbrONB3hFeZCIiACIiACNQiEMUmRCQEJ/4QlRjohGUIUXyyTCYCItB6BMZqvS6pRyLQkQDeS/ywEF5TjzFKFH/6IaqHmsp2hkAUmfbee++SzQm9wksDUaUrhrcS9u2333aohhHZxhlnnA7L44IrrrjCw/gYqazeoeRjHdnPrvSFEDREJjyW4ogpjJ525pln+ohzhIsx8l1R60pfsm3QH44dXkgMVUw+rdFpl19+uYfbDRgwINx6662paDk6+1St7VrHgW2rnaPV6tY6ERABERCB3kkAEUlCUu889trr9iYgkam9j1+v6T0PnfUKTFk4eD9JaMoS0XSjCRC2SX6khRdeuKRqxCfyD/Xp06dkeb0z1I19+umnHTZlWbkwOgr+/PPP7g0z00wzheuvv97/WB5zGeHZxDq8q4paZ/sS649eXnGez2mnnTaQzJ8hgnlDSbhaEetqX2iDvEebbbZZuPLKK11oOu644wq3X6SP9ZZB6CIxOzfW1113XZh00knrraLby9c6DnSoHfaj28GpQREQAREQAREQARHoYQTG7GH7o93pgQQOOuggz9/S1V0j74tMBJpFADEJkSQrjuB1hCcTnjpdNZJ7YzG3U6yPNki6PN9888VFJZ946Iw11liBnDgXXHBB+vfoo496OUa6w9OpHutsX2iDUdIQcfL7gbD0+uuvexhhlmGtfnWlL9SNVxXJ2W144nD66ad7OGE97dfqX73rCbVEVCcf1G233dY2wgwhm4SG5o8r+8+y2WabLYw//vj14lB5ERABERABERABERCBNiMgkanNDlhv6y5hckcddVRDdnvEiBFKDtgQkqokTwBx5OOPP+4gJo0cOdI9icqNuJWvo9Y8eXkInbrssstKig4fPtyFEobXLWd4DZGvKf9Hwm2MkcuefPLJcptWXNbZvlDhBx98EPbdd99wxBFHlNRPKN/bb79dMhJfSYEKM13pC1UyAtztt9/uXkw777xzhVa6Z/FZZ53l1yhEJhLIt1t42eDBg8MNN9xQEtLJMWVUw0rnZ/eQVSsiIAIiIAIiIAIiIALdRUAiU3eRVjudInDHHXekCZM7VUFuoxtvvDG3RLMi0HUCMR9T3mMpLs+KTCSzRBgp5/FRrSd41+CNd/fdd3soFUPCX3vttYGE3htvvHFYZZVV0s0RcGjjlVdeSZcVncBzkG3feuutipt0pS+EgDGSHGLZoYceGp577jn32Nluu+08qXlWfGp2X6K4NOusswZEaNrL/jFiZTRYw4XwtUbYzTff7PVF0RAR8MADD/ScWeSqyvYjTo8aNcqbrqcvnBfZcyP2vdLyQYMGdShftD1yWpEfjCTuCKwPPfSQT5OHi2MdLb/vcbk+RUAEREAEREAEREAE2p+AcjJ14RjiZYPxycMKYSnLL7+8v33mQYo/WdcIkEMGro0yHnxkItBoAuRjYvQ4RIissZzR4BAxojG6Gufh999/HxcV/hwyZIiPxoZQddJJJwVClNZcc81wwgknlNQR2yiXJLykYJkZvLKK9K+zfSF0D6Fmt912C0ceeaT/wQ4h7pZbbgkIEtGa3Zco8CDYlfOYJJdW9G7CUw0ujRohjTxa1BdHBSS59+eff+67fswxx0QEJZ/77befJ0uvpy+VctlVWx5HQoyNF21vjjnm8CTlW265pX8X8KJbYYUVwtlnnx0mmGCCWJ3nFcvue7pCEyIgAiIgAiIgAiIgAm1PYAzLg5G0/V6Mhh1Ycsklw4MPPugt81a/EkaG3OSBUNY5As3IjaIE4J07Fr1hK863VvjO4rHCiHQkf84LSByHn376Kbz22mth5plnbsqoY4Tl4dkz8cQT1zzsXekLw96/8cYbPvJepba6qy81d9QK4G1F2Ndqq61WpHhTy3R3X+ptD4EQgRUhtNHGCxyEL/KddZfxMkkDSHQXbbUjAiIgAiIgAiLQzgTGbOfOj46+c6PJgygjIEWrJDCxniSu9YzaFOvsrZ94d/DwfN9995WEV/RWHtpvEShHAG+g2WefvSkCE0nASaZdSfTJ96crfSFR9DzzzFOxre7sS36/8vMIGuSNItRvdFt396Uz7TFiYTMEptHNXu2LgAiIgAiIgAiIgAhUJ6Bwuep8StbikYRohNUTwnXssccG/qqJUSUN9dAZwkzeeecd/2M0rHLTvJ1uppHrBKGQN+EyEWh1Aoz8RlgToWkbbbRRt3S3f//+oVVyl7VSXyaccMJw0003uXdOtxyIKo10d1+6u71Kuz5s2DC/fj/11FNh2mmnrVRMy0VABERABERABERABEYjAYlMBeFnBaaCm5QUQ9zgDXjWA6qkQBvPENqDYJQVjvLzrMvmoCE/Bw8J8Q9vhjjdr18/nyYxMvk8GmkxcW4j61RdItBoAn369PGE3rHeySefPE42/XPxxRdvehtFG2ilvuCZ0yrW3X3p7vYqceY3grxP/PXt27dSMS0XAREQAREQAREQAREYjQSUk6kg/EblBmq3fECffPJJ6nGUF47ifExcG1FOPfXUqWDEQ0EUjaKIxCe5OopYo7hn2+rtHmVZFpouJdAqOZlKe6U5ERCB0U0AD1jlZBrdR0Hti4AIiIAIiIAItAMBeTIVOEoMG94o4ya1FUSOH3/8MRWPqnkgfffdd+mujz/++Kl4hHA055xzpvNZMYkRhRpljNZFAmKZCIiACIiACIiACIiACIiACIiACIhAaxOQyFTg+JQb3anAZhWLNDsn0GeffZYKSHgbxb+smPTBBx+U9G+qqaZKPY5IKLz88su7gJT1QurOkJ3YueOPP97fHsf5rnwyLPe+++7blSq0rQiIgAiIgAiIgAiIgAiIgAiIgAiIQAUCEpkqgImLEYQabQcccECncjP9/PPPqWAUQ9WigBQ/EZK++eabtMvkgsqGqeFJlRWOogcS5VrRSNA977zzhmeeeabL3SNZ+9FHH93lelSBCIiACIiACIiACIiACIiACIiACIhARwISmToyKVlCDqVG29dff92hyi+++KJEQIqiUVZMeu+990q2m2KKKVIBadZZZw3LLLNMOh+FpCmnnLJkm3acOfXUUxvizXTYYYe14+6rzyIgAiIgAiIgAiIgAiIgAiIgAiLQFgQkMtU4TF999VWNEvWvfvbZZ8MWW2xRIiplhadxxhmnJFRtqaWWKpmPnknjjTde/Y234RZ4MyEQDR06tNO9J0yOEQJlIiACIiACIiACIiACIiACIiACIiACzSEgkakG14cffrhGic6t/uijjwLDQi+xxBIdvI8YnU1WSiAKRJ0Rmg488MAwbNiw0go1JwIiIAIiIAIiIAIiIAIiIAIiIAIi0FACEplq4CSH0SOPPFKjVP2rb7311vo36uVbIDTxx4hzI0eOrElj7rnnDqeffnrAE0omAiIgAiIgAiIgAiIgAiIgAiIgAiLQXAJjNrf69q990KBBDd+J7bbbruF19qYKR4wY4eFz++23X5hnnnlKdn2uueYKiy22WCCXFmGJEphK8GhGBERABERABERABERABERABERABJpGQJ5MTUNbuWJyKsm6RiCGzx1zzDFeEaMASlDqGlNtLQIiIAIiIAIiIAIiIAIiIAIiIAJdISBPphr0EC4WXHDBGqXqWy0xpD5eRUqLaRFKKiMCIiACIiACIiACIiACIiACIiACzSMgkakA2xNOOKFAqWJF9t9//yBBpBgrlRIBERABERABERABERABERABERABEWgfAhKZChwrRCHy/3TVSER99NFHd7UabS8CIiACIiACIiACIiACIiACIiACIiACLUdAIlPBQ0Lun0UWWaRg6fLFGOlMJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAI9kYASf9dxVB977LGw1157hZNOOqmOrYKPgHbaaacpTK4uaiosAqOHwNChQ8O99947ehpXqyIgAiIgAiIgAiIgAiIgAiLQxgT62Chdh7dx/7u96yuvvLK3WeshdNxxxw0///xzOOCAA8I111wTZpxxxm7vqxoUARGon8AYY4xR/0baQgREoMcT4Hdct0w9/jBrB0VABERABERABLpIYIzErIt19NrNudn85ptvwj//+c/wzDPPOIc555wzTDDBBGGNNdZwzyUl+e61p4d2XAREQAREQAREQAREQAREQAREQAR6FQGJTL3qcGtnRUAEREAEREAEREAEREAEREAEREAERKA5BJT4uzlcVasIiIAIiIAIiIAIiIAIiIAIiIAIiIAI9CoCEpl61eHWzoqACIiACIiACIiACIiACIiACIiACIhAcwhIZGoOV9UqAiIgAiIgAiIgAiIgAiIgAiIgAiIgAr2KgESmXnW4tbMiIAIiIAIiIAIiIAIiIAIiIAIiIAIi0BwCEpmaw1W1ioAIiIAIiIAIiIAIiECPJsAg1RdddFHYfvvtw/LLLx+GDBkSzj777PDjjz+2zH7ffvvtYffdd+9yf5588smw0047ha+++qrLdVWq4O233/Y2XnvttUpFWmL5OeecE04++eQu94Vz5ZRTTilcz7vvvut8XnnllcLbdLYgI4hzvEeMGNHZKkq2K9f3Dz74IC3zzjvveHuvvvpqukwTItCuBCQyteuRU79FQAREQAREQAREQAREYDQRGDVqVFhnnXXCNttsEx566KEw/fTTh0cffdQflJdbbrnw8ccfj6aelTb773//O5x33nmlCzsxh/CDKPLdd991Yutim8CMNt5///1iG4ymUv/85z/DjTfe2OXWb7vttnDTTTcVrufTTz91Pgg2zTaOM8eiUYJfvu/33HNPmGOOOdLd+OSTT7y99957L12mCRFoVwJjtWvH1W8REAEREAEREAEREAEREIHRQwCh4YYbbgjXXHNNWG+99dJO8PCMV9OZZ54ZDj300HS5JkQgT+Af//hHwBuuqM0999zh66+/DuOPP37RTVqmXL7vzz//fPjiiy9apn/qiAg0koBEpkbSVF0iIAIiIAIiIAIiIAIi0AsIEEY09thjh5VWWqlkb/Fiwrvp22+/LVn+0ksvhbvvvts9Q6aeeuqw1FJLhcUWWywtc/HFF/uyzz//PNxyyy1hkkkmCWuvvXaYYYYZPGSJZbPMMot7T00wwQS+3bPPPhv4W2WVVcIVV1wR8HChPwMHDgx9+vRJ685P/PTTT+Hqq68OTz31VJhiiinCaqutFuacc86SYggACGmUWWKJJUrWVZrBW+Wyyy4Lr7/+eujfv39YccUVA+IC9tFHH4Xrr7/e+z/llFOmVdCPaaaZJiy55JLpMoSXK6+8MowcOTLMPvvsYcMNNwwTTTRRup4J+nXrrbeGscYay0W+N998M/z888/eJh5R1113XVh//fXD+eefHyaddNKw0UYbhd/85jeBPuKF9OKLL4bxxhsvzD///M55jDHG8PrvuOOOMO6444a+fft6HWOOOabzmWuuuUraZwavm6uuuip8+OGHfixXX331EOvpULjMggceeCBwLOAfLe4X/Oebb76wwQYb+HnGes4NeK2xxhreP449Ys2aa64Zrr322vDMM8/4ObLpppuGeI7EehE/ae+XX34JSy+9dOA8zRohnvfee2/417/+FWaddVbnmF2fn0Ygm3HGGcPiiy+errrkkkv8uGfr5nwYMGBAmHnmmdO+w+vBBx90ge2vf/1rSR1UxnGlr/369fN95TsgE4G2ImAXMZkIiIAIiIAIiIAIiIAIiIAIFCZw11134YKSDB48OLGH/arbmdCRmKCRLLDAAomJHYmJTImJEYnl9km3++1vf5uYqJRMPvnkiYkziXmrJBaCl5x22mmJCQa+jHUsM6HAt7N8PokJNsk888yTrLzyyomJHN6nrbbaKq332GOP9e3jAhN7koUXXjgxESoxcSNZdtllk3HGGSe54IILYpGEMvZgn5jYkpiXlk+bGON1Wx6dtFx2wgSXZLrppktmmmmm5A9/+ENCeRPhEhOLvJiFEvr2jz32WHazxESoZIcddvBllvfJy8w777yJiUvOg35aWFViOXvS7dgn2LOtCTGJiUg+baKSlzEB0NfDA44mFCUW9pU8/vjjiQlaiYkjycYbb5yYsOblNtlkk7TuddddN1lwwQWTqaaaKtl88829ftqy3FslZeBjIkhiAlxaz5577pmWKTLB8eZYRzvmmGO8r9TLeTLZZJMlv/vd75LI3EQk768JQb4Jxx/mJtAlJop5XRwz+JloFKtNOB/YB447+8e5aPnD0vWcTyZUeRn6Q7l4vE28TMtlJzgv2Pdob7zxhm/P8YhGvznPTSBNsn03Yc6PF33iHDSPwMTENd9+kUUW8X3hfObYcRwsB1WsUp8i0BYEUFBlIiACIiACIiACIiACIiACIlAXAUv+7GIND8s87CNK/P3vf08saXJaj+W2ScxbKNl2223TZTzUIwzxMB8NkYlyb731li+Koox58CSWo8iXWX4lfxBnHYbIQNt//OMffZ5/5i3lD/bmaeLL8iKTJSl30QlRINrhhx+eTDjhhIkl3vZFlmvKhZMo7Hz55ZepKBAFj7ht/EQMow5LDB4XuXBhXjU+H/eniMiEyPP999/7dpYI2oUG+o1RD6LR0UcfnYpttA2HvMhkHjuJeZQl5r3k2yI6IVhlBRjYIYSwjxgiDHVFcYxjNXToUBdmEKpiGbYZPny4z/OPY48oYt5U6bJaE1mRCQEMQe2www5LfvjhB9+U4wFT6sayQg3z8fjvv//+iXlEscj7Tf8t35PPcx4wj7ATDZGKZeZZ5os4jxEaH3nkEZ9nn7fccksvU0lkQnQzL7LEPK58G/NISiaeeGJnaZ5KvuzCCy/0ZRzLfN9PP/10L+sF7V8UmcwzKuE7g8GbfnKsZSLQTgSU+Nu+uTIREAEREAEREAEREAEREIH6CDBqm4k1PkKYeSl5jiZClUw8CoTHYYRkPffccyWjkZHYmlAjwp+yRtgbYWYYdbCteXQEwusw81DxMDgSjWfNPIfSWUKVCD1jVLlyZiJYoI8kKreHNv9jdDwSPd98880+f+eddwbzLvJwJeow8SDsuuuu5apLlxHyxohkJlh5KBt1k7Pq0ksvTcsUnSBEjJA1jBDBrbfe2sPBmCecDdtuu+3S0DTCE2GVN/Pg8fxFJuD5KkK87rvvPg+xYwEj5ZmnjO+ziUzp5oSarbXWWj5vYlLYb7/9PKzt/vvvT8twnGIZFhKyBsP//ve/aZl6JgjhI6zv4IMPTsPjpp122jBo0CAPyTPhp2J1HJsYHhlD1WLydPgTrkbesHi8CZcjPJIQSwymhOwtuuiiPs8+H3DAAT5d6R+hgdRnHn1ehDo4Tia0ecgdC0lqvuqqq6bHslJd2eUc13gsCbEjTDF+l7LlNC0CrUxAIlMrHx31TQREQAREQAREQAREQARamIB5MIXddtvNBRVyAZEzB/GIfErRzDPFxRfy11gIVLCQMh+RjhxCWbMQrHSWXEDmXRIQr6KxjDxQWcGB3E3Z3E6UpX7zHImbpZ/kEEIIIg8OdcU/ct9QJ8PHI5qRXJoH/KzNNtts2dkO04gYe++9dzCvomAhXgGBxLyEOjVSHMJa1tgf2CIKkbOI+sklFQ1RKJvTKS5nu6yRv4m8QeSt4riRo+mss87yItljYSGEJcIIgheMskzzdXNcMYSmzhhCCoIafcwaIo15Anm+rezyOI0gRP6oaAiCGDmWMI7pyy+/7CJUPN58vvDCC76OMuxX/njTF8pVMvhzPiNmcu4gNiGSsozcY5zzrGMExnqsHNfOMq2nXZUVgUYSKP0WN7Jm1SUCIiACIiACIiACIiACItAjCZB8mYTYWY8PBAKSTZNcGu8gC31zLyAevPHwsJA53wYvpV122SX1AomAovdOnC/yiQdLXpjAKwfPorzRB8xyB7knUH49QgllEC4QmrIWRYvssuw0gsTxxx/vnjh4tVi4lnsxkbD8P//5T1qURNdZy3oQxeWIa1kjSTdsLHTME4AjuuStXP/yPEn+jScTnj977bWXJ0jH28bC0Uqqy2/HyjzT6DlUsmEXZkhsnmceq4MHHld5zzfWc6z4q2QcTwQ4y//VoUj0GLJcXx3a5jhlxcwOG9sCPLkYRZEE7fAhmf3TTz/tydbhzHHKJjUvV0d+WaO55uvXvAh0B4HK8mx3tK42REAEREAEREAEREAEREAE2o4AHkR4weBhkzdGecOjBKEHrw5CqAjV2nfffV1kQsTAIyfrPZOvo+j8Z599VhJOhAeJ5T0KCy20UIcq8NzBw+hfNoIYYUjxD/Fm2LBh3k9CthhdLBsaRkWWC6lDfdkF7B/CGW0wGhyjup144omBUd/wpqEN7JNPPkk3YwQ1yzuUzscJy08UJ/0TwYpR6hCy8NrCM+d1G8EuGseA0f6qGWUIB7ScR76veAjRV0QRLHssnnjiidQTiHWM4obAU44p6xth7B+c8jwYFY7jlBfeirZpub/82CFSxeNN+Jzlc3JvOupB9IyjvcV6OYdqGSITx+Hcc8/1UDuEMkYUtETd/t0gdI/zqZxFYYyQO5kI9DQCEpl62hHV/oiACIiACIiACIiACIhAkwkQIof3hiWp9nxLDP/O0Ot4yBxxxBFhxx13dGHARsvyUCWGuidUjZC1PfbYIyCkWFLqhvSSnEyEJiHg7LTTTu6BQm6bckbOHxvFzftpiZVdZNliiy1cpCHnE2aJpF0kQpCx5N8uSCAYVTPC6c4444xw3HHHBYaof/755z2PEkIbohXhWIg6Rx11lOeosiTmAc+icuIJOZDwhMGz59RTTw14Mh1yyCHePHmWCCtEwCA077zzznNPHdhWM7x16COCFfmKCL2zxNRprqzs9oiCMCWkDMGEnE/LWghduZC8Sm3CmBxFfBYxjhd9JF8WbbLvlhw7WAJyF++K1FGuDMcSIccSebuQxPHk/Lv44otdXGIbxEHEQNpGwLOE4O6JV66+7DLEKkuk7hwRlzDCO9mPa6+9tmqoHIIU/br88ss7nccq2xdNi0ArEZDI1EpHQ30RAREQAREQAREQAREQgTYggEcN3j2IJ4ceeqiLEIQG8XCN0PKXv/zF94IE20ceeaR7MpGAGoEEceqCCy5wLyi8V7piePessMIKgUTMU045ZUC8IeF2TCCer5uE3ghGeBoh/gwcODDglUVC8Bg+hahiI6p5iBW5i/ByQqyoZohpNpKd5zxCWMIzB0GD0Dn6SNgWog4iBt41tPv73/++bDjVQQcd5Im08QZjGo+xmNsHUQqvG7xv4Erf6C/tk5upkuE5c8IJJ7gQhzcXoYE2QpqHLBKiZSOrpZsiHCI64flDX8k/xHElH1ZRI1SS+vksYuwrCddHjRrlibqZP+mkk1y4I8yys0b/8eBCqEMk43wkvI1jgUCEzTfffJ4EnHNn9tlnD4MHD/Z8WvkwzHJ9wJuJ0DrOQYxjjQAI72xi9Py25N2i/U022cRFzPx6zYtAOxMYwxRU+ei18xFU30VABERABERABERABERgNBLgcQIxgZCwbELqfJcILULgKOe9ky9bZB4vH/IrEeqFKIL3SxyJrtb2sc+Uj+JSfptYBmEihjfly5Sbx1OIOvFcyht1vmHJxWtxYJ/wrkGUy+bpIZwMLyRGR8saiaoRohCSalkMZ0TIyRsJzAnjI5H1Rx995PtRrlx+u3LzeCfhWVYpzI4R6cglhXdV1gjN43jCqJGGhxks8TSqZPBFrCyXl6rSNl1ZTrgnImf2GHelPm0rAq1AQIm/W+EoqA8iIAIiIAIiIAIiIAIi0KYEEGCyI8NV2o38yFmVynVmOQJXzHtUZPsifS5Splxb5ZKOx3LUWYQDokN+xDPqwNsGcQbvHDzHEK1IPk1ycUY3K2KMFFfEEFs6a4QLEvY2//zzl62CpNoIOnhL5Q1xrpxAly9X7zx5mWoZnmvdaXj3yUSgpxGQyNTTjqj2RwREQAREQAREQAREQAREoEcSIGE3+ZLWXXfd0LdvX/dqwvPn5JNPDiuttFLL7DOeVQhh5ULOhg8fHoYMGeKhcYT8yURABHoWAYXL9azjqb0RAREQAREQAREQAREQgV5BgLA0QsoWXXTRXrG/2Z0kNJBR8gjLI7cTIVeNMHJGEaqXD8drRN2xDryuyOeFJ1al0ddiWX2KgAi0HwGJTO13zNRjERABERABERABERABERABERABERABEWg5AhpdruUOiTokAiIgAiIgAiIgAiIgAiIgAiIgAiIgAu1HQCJT+x0z9VgEREAEREAEREAEREAEREAEREAEREAEWo6ARKaWOyTqkAiIgAiIgAiIgAiIgAi0PoHrr78+rL/++mHFFVcMzz33XEM7/MEHHzS0vtFd2RtvvBF22mknzyFFX26//faw++67V+3WqFGjAkm9o+29997hlltuibNd+qzFd5999vHE3V1qRBt3isBnn30W9tprr7DccsuFAw88sFN1aCMRGJ0EJDKNTvpqWwREQAREQAREQAREQATakADD02+44Ybhv//9b/jd734XujLcfX7311prrXDOOefkF7f1/IcffhjOPvvswCf273//O5x33nkV9wkRaN555w1PP/10WubCCy8MI0aMSOc7O3H88ceHzTbbrOrmF110UXjiiSeqltHK5hDYc889/fyfbbbZwhxzzNGcRlSrCDSRwFhNrFtVi4AIiIAIiIAIiIAIiIAI9EACzz//fPjpp5/CxRdf3PCRyB5++OGw8MIL90BqxXfp448/Dgh5zTBGdmMEuWr21ltvhbHHHrtaEa1rEoGnnnoqrLHGGuHcc89tUguqVgSaS0AiU3P5qnYREAEREAEREAEREAER6FEE7r///jSU6rbbbvMQsFVWWcX38YUXXgi33npr+Oijj8L8888fNthggzDWWKWPHGz/2GOPhffeey/MOOOM/kDNJ3bBBReE77//3r1oLrnkkrDFFlsE2sBiG0y//vrr4c4773SPnPHHHz/885//DBNNNBGrPBSNsgMHDvT5Wn369NNPw2WXXeZ19u/f38P/5p57bt+Wf9dee61Pr7feeumychPV9qtc+UrLYHfVVVf56htvvDH88MMP3qdYHi+om2++OYw55phhpZVW6iDIvfTSS+Huu+8Or732Wph66qnDUkstFRZbbDHfHJb/+c9/wjfffBP++te/+vH57W9/G6tOP6+++uowzzzzhAUXXNCX1WKUbvjrxB133BHGG2+8MMssswTCKvF4+/3vfx/WXXfdkqIIlbSFsDLFFFOE1VZbrYNo+d1334X77rsvPPjgg4Hjsuqqq4ZJJpmkpJ577rknPPDAA+GXX34JSy+9tIeaZQu8/PLLfhw/+eSTMGDAgLDOOuuUeN/VWk8/hw8fHhBXv/3224CX0UYbbZSec88880yAO+IoXmDzzTdfmHjiiQPtDR48ONuVgMjHMfjDH/5Qspx6//73v4f3338/cEw4PrGfhE5efvnl3j7HlPN7rrnmSrevdv6nhTQhAt1FIJGJgAiIgAiIgAiIgAiIgAiIQEECp512WrLAAgsk9rySmMiRHHbYYb6lhbgl44wzTjL99NMn9mCd2MNwYsJCYjlm0pq33XbbxMSHZNlll03WXHPNZMIJJ0wmmGCC5Mknn/Qy9lCdmAdNYg/xXgcLTVRITHxI62DCRBhv34QqX05d1Eld1GnilC+v1Se2n2666ZKZZpopsYf+xIQxb//KK6/07fln4YD+ly4oM1Frv0xY8P6auOZbH3vssd7XMlUlL774YmLCkJc3cSgZNmyYFzPhwfvB56BBg5Kpppoq6dOnT3LNNdek1Zx//vnOl+NjIogfgzHGGCOBA3booYcm00wzTTL55JM7U8sVlW6bnWD94Ycf7ouKMMpuy7SJSYmJPUnfvn0TE1ySRRdd1PfH8lKlRU1MS0yU8X3g+HL8OH9MaEzLmPCSmFiYsA8rr7xyYuFjXsaEtrTMVltt5XVTF+1yfg0ZMiRdb4KbH9OFFlrIjzHHG3YmCnmZWus5f6l7sskmS9Zee+1kySWX9D6bgJaYAOZ1nHDCCYkJpYkJP37+MW3hkd7vPGP6sfXWW6f9ixMmSPkx4fydYYYZfJo+wn/mmWf2ukxcch58R04//fS4qX+Xyp3/aQFNiEA3Egjd2FbZpr788svEEtglBx10kP9IbbrppgkXXS463WmWXC2xNwKdavKuu+5KLHY2saSH/mN49NFH+8UgX5mp0vlFPXZ+5MiRyY477phwfNvV7G1B+uPayH34+uuvnY3FuTeyWtUlAiIgAiIgAiIgAt1GgPskRCYejDHzzPCH+1133TWxUCxfZnmF/GHeElz7/OOPP+7b/O1vf/N5/vEAjVBiiabTZebRkgwdOjSdLyoy0R/z/PH2v/jii0J9QjDjof6rr75K20Ow4pkkmnnZJPxVsiL7VY/IRDvPPvuss7r33nvTZhGXLPdVYh5KvuzHH3908QHxBUPwgB2CVzTz7EnMI8lFkrjMvMsSSyodZ8t+ZkWmIozylSD2cDyyz1d//OMfXeyJrLfffnsX2rIiDMIWx+Ptt9/2KhGLEMXM0yhtYplllkmWX355n//HP/7h7SA6RvvXv/7ly8yDyhdZcvqEbaJZMvVk1llnTdgvrNb64447zoWtV199NVaRIEKyfzfddJMvQ2Rift999/Xzzzy/Es5B87JLRUIKWoJ8L5c9rmmlv04gpPE9ioawhWgVRTGW77HHHs7SvPS8GOds/vyP2+tTBLqbwGhN/E3iO1wKSRqIyyyJzXCH/Mtf/uKJ7uzLZ9+V7rHOJtI76qij3H3VLjTucmk/lMHeEARTqEuS5eHC2ZsSt+GeS3JDjme7Gi7PV1xxRcO7DxPYwEg2+gn86U9/8tFeGPEl/7f//vs3vIP5kWIa3kADKqw14kwDmmiZKviO24NMy/Qn3xHCGTgvG5HoNV+3eQ143Xazn1/VlPnedF41BaAqFYEWJ0DIk4kegRHQzOuEF9l+b8zoc9wnY4RemaAQTMDxecowepp5u5SMouYrO/HPPKg89I4wMsKpivTJBAwPHTNxw0O26NMNN9wQLr300rQHhD7xV8mavV/ZdgkVM68WX0QYImFThGlhhKcxyt/JJ5/s8/wj9MoEii7xLcIobTAzQUgZ4W/RGC2Nc4QQMozQMM4Fjhvc+TPhyZ8fCAfE7GV+2GabbTw8zRfYPxOWwplnnumzHCfC3whljHUQLjfnnHOm9/H0n988E5UCuaYmnXRSz3e1yy67eB211pvg4/fthP5h7EMM18uO/sc6e8nuYYyEu1GGfrGf0Uxg9eNHCGMRoy1Y7Lbbbr6fcRvzhPJ+XHfddXGRcySXUzz/0xWaEIFuJjDaRCZTdoOp7h6rSgwrw3ieeuqpwdw9PdaUiyEXlFpJ6bqZV0lzxCYjKHGBMrXdL3imnHt8OUKCKdlpeeJ32WeZCIhAaxEgbp5cC+QCyP/xvW2klRspppH1N6KuIiPONKKdVqmjWWJyo/avmaJ0d74M6IkjRTXqGKseEegpBMzLw+/buYfnITf+nXXWWX5vTL4ZlvGSmQdkRqQzjxXPm0SS60bc81vIWwnOIn1CBEAYQ4CgT9NOO20wjxsXZ0oqqzLT7P3KNp3fRwvhKnmpS/4mBLPFF188sI7yDz30UJf4dpZRv379sl335z4WIJzwYj7mhYrnCp9sQ14ljh0vQci9FcWdWBllZp99dp+lHPmUzBsuPeeohzxcrMOOOOII54FQY2FonmvKQhC9H0XWjzvuuM6QfFIIWhaS6ecw22bPW/qAYJY1znXuJy3Kw/cLUYw8YwixRYzfavJB0W7WED05V7MvrfPnRra8pkWgOwmUZuHrxpbN7TBYvLGr7fkvhMXIhpNOOinw5oNhOy2mOJRLpmaug35BQb2vlNyOXWLUCy60KPkkoePHD1Ud1TpvtRLpZcvzQMqFZfXVV88uDuZe6m/GEc64iHJhIVEd6joJ3OhLTCZIn0jsh6pubpuudmeHgOUB2Nw7/cJGwjwSzPH2olaSPMqS2G+FFVZI33aUdNJmiibkq5VojnoR0Eg4B5Mlllgi35TP1+ozhfD44o8fF5INcg7UGtmiyPG32Gn/oYIJP7689SH5YNY4Brwp4EbHXE6zq6pO099qiQY5B/DKQ4DkGFtYZYf6KEMdnMcknCRBoLnfesLMeK6wUa22aiUt7NCwFjgBjne1YYQbhamZI8U0qo9FRpxpVFuqp/cQ0EhRvedYa097LwGSbyMacc9b7t4tPqjjzYIXB16a3DNyn5N/gC5HkfvIrFlKhuysT9NG1or0CUGCFywHH3yw3xtzr4gQYOk8PDkz62sZIk5n96tW3fn1CBmVzHIH+XMG+20hc86XqBFeiOMR1FnrLKNqfaWPmKUcCdttt12HriGQxfMIgTJrPH8h0tAv6uE+33JOZYv4NJ5dGHWRjP7NN9/05yPu93EU4BmC5PK11nN+4DyAYIRgxXlL25bbyeuP/9jf/D5bWJ8LW3hf8bz0zjvvhC233DJuUvMzJrNHkMsbz5Y8N0TLn/9xuT5FoNsJ2Mk5Wow4YBLaFbVyydSIMa6V3I76iV9e1hLJkXiOGGWSENoX1mOdY/uUIakfn5US6cWy8ZPcOnZhS8xFNrGH/8SEgriq5JMYYfsB9ThZktqZC66vtxExEht1wGORiY0mNpg4axMb0u1ZbyKWx6qTxI5EeEWS5JHnyk4mjxdOK8tNFEnIVyTRHP0hOZ1d2BJ70+HTJE2kfWLxsSJ9PvLII70OYqzpm7mYevx4Ja7UW/T40y/4EY9Nkj77YUiI4Y5mN0Qea00SQI4F8ezmZut/sUy5z61qJBrkHI0x0uTsImlgZGNhOl6l/VD6MYYX3wvOT5IG/uY3v0ljxSlYq61aSQvL9V/Lkg65C6oxgbHdXHiuLqbzxrlqrvlexkLt/PyMeQfszW1ymCVG5TiTd8JEXt+cc9jE2ZKquAbE84MVFs7reSXspjch55u9tUrL1+pTWvDXCRNZExtBx3MGkEiU61A0uwHz65m9HUxs2NyEfALRuKYccsghniMjuw3r7WWAJ2C1t42+jyQgtYcB35TvPslGLSzRE2AynzVyVNCWeexkF5dMm9ie2NtITwrL/pOng+3yxvXKPGKdL8k2YZ43u8FMzMXejxFJZnfeeecO3/Nafc7XyXy1Y896e1Hix5QEphzH/fbbz/fbbhpZnRrHh3ODPIUcd3th4OdM9nxIC2cmaJ/jynlnN8OJvdzIrP3/SXJQUM+BBx7oOTJi0tx4naYUZVh+wAEH+Cfz+XOhSFvZxjnH+c3lWmgvfdJVteqxFzWJjSaU2A15csYZZ/i+mddhun2c4DvAuQkzGMAwazZKVcI5RJ4M7iXIh5E1e6D0312+dyTEtYdFX02eGZKqmodDYqEnJfcM2e01LQK9lUA+J5O9SPXrlY3AVYLE0mD495OFJnYkFqqU5mximT14+31uTNTNMu6H+c2Mxr0ZyZKzRj5VflO59mNcY7iHz1qRPlkok/8WlNvORIjs4orTRfar3pxMMXcPzxjReE7hPiRrJo75vSPLuAeBST7v5yKLLOLJ2ON2lqrE74njfLnPbE6mzjDiXj7mTYr1c72lf6+88oovMk8cTyAf1/Np4ZTJJptsksQE6dy3b7bZZtkinr+X+2V+Q8mdRRLs7P0Fvy/ke4q5v/h94Hcka2zHsyFWaz05rSwMLbu5/46yLzFJOb8vJC0vZ/y2kLibPEs8k9aybE4mniX4zpgoW7IZyeFJhh7zTpU7/0s20IwIdCMBvGu63Xj4IKGbuSwWbrtcMjUeSookt+OCzJeem3WMCxJiDw9S0ShTLZFeLJf/JGmbqd9+wUQU4Qt+yimnpD94sTw3qlwIovHgycgGXLBiwkRuhBlBArGJaQyRiYs8F1z6zfIiSfIYBYEbZfOciE12+OTiz8XR1Px0XT4hX5FEc4wCgiDDgwBGsu8oqsWHlyJ9Zj+zNxQWxpLw41PuYZJ26jn+nCc8XGLmmeWCIqwx6jGvNk/aHh/4eEBDNGO/KlmRRIM8mHDuPfLII14NPxRbbrmlc48PjfHcRujC6A8/rhybmJCwSFu1khZ65frXgUD+GtKhwK8Laol8tUYeqTRSDEI1N+BZQ/jIXp/Kic2Ur9WnbJ1Mc/4xChDXLL7/iJqWzyGJCVm5CeK7wHeRayTXHURQrlOcjwjqTLMNN2fUh3EOm4doyYgqrOO85yaQ/jPCDeIp9Wdvfnnwp+5ygpBXbv+4PtcSiouI9kXE5CJ9jv2Kn7WOPeX4XeCaz4g03NQiOnONmXfeedMXFDDjNwQelUTp2Gb2kxtrBGx7e+rHjRvY/Og8lJmhwMsARoSiXxx/hHnzvPX+IJJhRdrK9o3pciNFFamHc7TWyES1Xk4UeVECc5hlR6RiO46XeVpXHG0qv5+aF4HeRiAvMnH/wm8B907c43B/i8jD/S+iOYbIzjUOwRghgKTFfP9Yxn1MNL57/F5xL4jxgoEyvLQgITQP9vy2sKyayFSkT4gZ1MNvMfetiDskGuf3it9AjPtT/ipZkf2qV2TivpV+8WKKJOBYLZGJbfgt4EUGL8Pfffdd/42nHu53oplHjv8WMXhRpUF6siJTEUax7vhZRGSyUErfRwZQ4mUSL9247nMPxLHDOIe470AI4tgjVFm4XGJ5cX09v0+s53eLlwWU4T6K37KYFJvE3bzsQABF1KQcv3eIbVit9eZp5QnsOTd4kcHzH32AaxzhrZrIRFJ8vgecs7z0qGVZkSn2DyHNvO79uNIPhEPEL14GYRKZHIP+tQiB0SIy8XacLyUXi6yRnZ8HEv7wKOGPiyQWH8T5kmaNH4PoKcByLqY8BCHUROOCbGFjcdY/uajSBx6iMMpk36CwDMGFH8taRvv8uGy++eZ+U0q9KM7RY4nt8yITb14pxyhsWeNBhOXxTSoXo3y/EOgYaYEHkvjHfuOdE4cnzdZZaZqLP8PDZi2+2YYLF1Eu2ieeeGK2iP8A0Ecu7rTPRRsxJWt4EVAmikxF+owwxY8KfeChrYgVPf7ZYUyplx+feGzjiCDZoVApw9C71UQmfszMvdtvQOJx4JNtGAIXwwuNB6ysITbAJopMPEwytGvW4vkRRaYibbFPCJ14ckRBLVunpssT4KaL6wXf3/xfvMEpIvIVGXmk3EgxRUWmvNhcpE/5PeZGjHMv+1aUG3fEn7iv+RFnLHTZb1jxZIrGWzPqwUMFi9dnBCNuyLnG8/BAvQxdzY0uxjLOd8SqeOPOucr1jnWVjOtzNaG4iGjP/tUSk4v2Od/PIsc+XtvxNIpeXnFkGm6YsSKidL5t5ouI+EVeBnCDynUxvjBgRFSOIcc6ikxF2irXR45fdqSoIvXwG0Xb1V6E1Ho5UeRFSRT2siNSce3ldyt7f0E5xFWZCIjA/xPIi0wsxUMFr3EeqPn+cl+Hp000XvTxPeI+mYdm7lsYVRoPDwv7SV9eIPggllMHIgjCAN/nWC/Xpui1U01kKtInytAHHthj/dyXxese64l24K+SFdmvekUm7im5dtOn6MVVS2Sif9yfwxK+/CEoIcrBMnoQMfoaHvMs49pXzrIiE+trMcrXUURkYh95zuA8oC+I/TzHZV9GUYbzgWsyZfjjxVO8t6BdS9nhnkKs43lo4MCBJRELvKS3XL/+8poyPN9YrsD0eaPWejzaED0jU7ySEKwspUv6u1BNZKKPiKn53xWWl7O8yMR9w5///GfnQ/95CWSpP9LnLOqQyFSOpJaNLgKjRWRiZxE3+HJkjaEpcfOPf3wREXowvrgo8/HmPG6HdxKu7FxMuPCiWvOWnjcg0Vie9xRArOJLGi+slDks94Yi634a6yryiWjBgxU/CvFmPS8yReU+e4GkbhjQr+jeiciU7RdiEusr/RGSUtS4+HMjkLWsGyvqP+3EoTmz5fAwIvQwcswKapSLIh4iUNE+I/Ig2tAmF3/eZBD6UM2KHv8sQ+rjQY99wOIPLzcwWSP8oprIFPta7ljwdgHjLQnu3FnjQZYfwCgy4UGXL4PAh/gQRaYibfHGkO9U7A+iHT9I+dCRbF80/f/hcryNQvzI/xHahBUR+bhBid6SbAP3+H1ChMa6IjLlxeYiffJGM//wFuK6hPjJd5Sb4rzlRSa+h9yIZQ2BCNEghiVEkSn7EiC+9WTfuUGMf5z3nKNFQxBol+tzNaE4irLVRPsiYnJn+1zk2EeRKf4msF9488AivtUsIkqzXd74raz24gH2tV4GcF3ipjXvYYx3G32MD1u12sr3Lc7nRaYi9dR6EULd1V5OFHlRQh3cmHOtzlp84cL9Bb9NMJSJgAgUJ4BAa7kuK27A78/rr79e9btFmfy9GV4b2etoxQbKrKjVJzZBsCr3opPfd8SPWlZkv2rVkV9Pv+OLoPy6avP8Jpf7nY/bcF2L0RRxWZHPSoyKbFupDH3hBXe1/eTeA2+n6LlTri6eO6pFcbAN5x2/25Ws2nrODV7AdMZweEDw64rBgJQJ1frflfq1rQg0isBoS/xNwjQTeDwJs72psHvY4ENTMrpDNFPM46R/kkiNv2j2RS+c3M7Ep7iZf8ZkgfZmO12erTtdWGXChAtPXM7wqFkjwZ49WHkSQwvh8BEEsuuZtht+X2ThWZ4kMb++UhI3Ex68aLUkefm6qs1X2+dsH/N12AnoieZIlGcPrcHEspIi9sCSzhfts+Uq8gR8DDFqD+eBITnNIyiQ7I+RBvNWz/Gvtp/2psar5ljYm5S0mew+pAszE+xXrUSD1J1nY0KpJzaPVcHZfizirH+aCOXfjbiwSFsci2pJDWNd+uxIgOGAqyX+ZnSSOHJJfuv4PeEaYx45wYRRvy7YTYoP3Ux5uynIb1b3fH6AhCJ9yjdigmZg6FxGV2FQAPrOKJ/mMRpMGM0X93mSYpqHV8k6zk8TmIK9lU2X8x2zB/V0nv5hlkfB/9IVv06w3sTT/OKK84wkkzUT4NNRYegjlq+P42qhgD7ySvw+5zkyvDIDFmCd7XPRY8+1kmG6o7EPWOybCTnBRL642j8ZUQfelcxu9tPReSykoEMx9slu3P06FIe8joXY92ic31yHTGiNi/yT40ySU6xIW16wxr966il33Kk+MmNYaAbEsJCHYC8n/J6AZMIMntCVEXnsLXk62pSJqH7czKPAE8Vm7xtq7KpWi0CvJcDvS/x9LAeBe3/znCy3Kl1GmfiMEBeaB5QPPx/n6/ms1SfqqvT9ttDcsr9l+faL7Fd+m1rz1ThW2zb/e5cvy28S9471WiVG9daTLU9fLKQ7u6jDNL+F+VHm8oUYPKqW1Trvqq03769a1Zes5/6PP3sZ5n8Wkl+yvt4ZGOR/y+utQ+VFoDsIjDaRyd7KBwv38BtqRJr8j4h5v3R48M4D4QtrbxaCuVQGcyNNV/PAkH+oY2QbHqyiMWy1uTwGy4cRF9X9aZ4wfvPNg9aiiy5asj39x+JDDxdPDHGGaXv76vPmruo3yD5j/+6xkcowBCY8VW4AABUHSURBVJdyxsWNdtnO3EvTIhZyEiyZa9hjjz0q/kCmhQtO0A4/5vk+8kDHQwLDvLLeQo0CPBGEomUfPov02d6g++gSlmPGR9NjREFGceBCilBXTmSq5/jHfpX7RBTE2AceVqJl9yEuy34y+p2FuwR+0Owtva9CQLKwNR81hb5TN6OsxONOIfOWyFYTLDeU7yPnbBTD2OfsOVykLUZF4YGMYXd32GEH/+MBn7pkXSNQROQrOvJIuZ5w3mQtiuDZZXmhvEifstvHaQtT8AdyznfEXM5hRvlB3DXPvVgs/eTmFgG2nGXFcM7deP5Slv5hiG4Ms5u37Lb5deXms3Xn18cb8Gqivb0N982qicmd7XPRY8+1P/4W5PeB+SKidH672OdqLx4oQ7t5wTsKNdQZGeYF72yZIm3l+1duvp56qh136q72cgKBDCt3/sYXJV7A/uW/X9zIc1y5rjISK98VXh7VM9pUrFufIiAC7U+A4e7jvV777432oLsIWPoAf06gPZ4P8iNbd1c/1I4IdDeByq9Hm9wThvk0d3S/cbNwjIDXEmKGxXe7UII4w80lwz5WMt68U4Z6uIlE+EBksbAI937JbkcZbhYtXMSH8LTcEC5i5G8ss9vUmrYEhS4WLbvssl43w2EyVD3ij7n5u/AULybcwHNTy/4hjCGiWGiEvyllG27sLZ7Y68FroJpSzn7wQGghVv6m1kZ2cm+pESNGpKIZb8QZZhMWnTUeSmgLDw/LveKMLSG2ezUgkrHfmIWe+Xo8u8yFOVhYSIkARplafWZ/LXTEhSa8QXggRHy05H3uLUQdeavn+Oe3zc4jzMCchzRLTOnDm+L9EAW/bNnsNPvNMd1yyy1dSGLfOf9s9CQXlyjLkLEIgDzY4ylguWyC5SLJVhMsN5mX4UGffYYj4lDWirQFQzxSLHQxIFIgbiEk4G0VjZsk2pPVRwCRD9ERQdHyQPgf1yjOdc5XjOPOd5rvCl4UiCjRQyYKhlFgMLfwtAMWNhTMXT2dZ8ISOpbMl5sp0qf8dgjyiMHmeu+eTJZLyK+H5k7v5wrl6WO2f3zXbYSvkmUID5xfCM2VLArpiKqRGZ94ePHwnhfWKtVTZHlsi9+QrMXvMEJEVkzOlsmKybGeevtc5Nhn26w0nRWlY5m8KB2Xx8+siJ/lzHnF953fm+zLgLgdn9l953zlxQK/Q1nLzhdpK7ttdjp7XnWlnmydFjLhv3P8hvKbaiPiBV4o4VGHuJ59UZLdLvuiJLs8O81LMK7f9BUvKb7X/LZzPY8eb9nymhYBEejZBCQw9ezj26y9w4Oce3t+pyxnarOaUb0i0HoE7CF5tBpDNW+88cZpIjYj5LkRGKaRfBXRKiVTK5LcjnwejGxEHijqt5tGzztB/G80ylQbEjSWy3+SjJAEhiQrp27+SJRHvqKYz4VtiN8liRvryfGAEQfNKGImlKXbkYPKHr58Pf/IycQ+Zo1+10qSR5Je2iKxbCUrkpCPvtRKNEf9JA+ODMgxFEcBiYm/i/SZhNgkdYzJ/8jZwTDa9oBeaRcKJTcsd2yzOZmo3EQ+P44kHIQbiYlJpF0tJxPb1Uo0SBmS1lIP9Zqo6aNMkXMq5mSiDCNaMeoU+24PxD7qCuVjYmXK1GqLfaiW1JA6GFGP81D2PwLkiuH7Ws3IR8MxIw9SpZFLiow8QpJrjmt2pBgLV/OcQyaIeI4zrg+cJ/nR5fLXgSJ9yu8TeR34nnLN4vtGvjQGVzABIB25Jj/ijAkRvp7hmbkmcy0zj0M/V2MOpErXZ67tXG+5XpEPijxQDOhgwmzaNa5R1Me1tJIV+Q6Tz4jRwExo8pwO5MPiWkQy92hMm6jsedgshMx/Bzge2e95kT7H+uJnkWNPTibzjomb+CffWdpn5ByMUXXshYQPHGBCSGIhw34eUCZ7vfDCmX8xx1+10Xm4lpjHcEKOJUbeIcm4iU/efrxOM2oNfYQBufg4F9iG9jnfsCJtZbqWTlrYRslIUUXqKfIbxXEn8SojB5Kng0TE7EMcqpqk7CRqrXdEHhP3fL+rjTaV7pwmREAEREAEREAEREAEUgJ4YrSM8QDGqESdsWrJ7bIPKLRhb+E700TNbXgA48GlmrF/WRGJsjxokMQtK3pVqyOuozztVUuSF8t29ROhp1aiudifavsRy1TrM8eH41lNXMrvT7Xjny9bbZ5+8QBWrxVJNEjSSI513hiGNN9mFCPyCdXZtkhb5i1Stq1825r//8TftUQmONUS+RAFao08wvmfHynGvAM9eTEP8jwck4QYQamWyFSkT+WOL4MdkMybB2/aRHQiyXE0RBqEIdbFgREYTc68QnwZfUSE5aE+WiWRCeEIASa2FQWmbKJRRqSjLUSoSpa9hscyeaG4iGjP9w+BrZqYXKTPsQ/xs8ixLyIyUV8RUTq2Gz85r2q9eKBsrZcBlEGMYlQlBG8S4TNaJceH6z9WtC0vnPmXHymqSD1FRKZaLyeKvCipNCJPvSMpZXZXkyIgAiIgAiIgAiLQawmMwZ7bDWSPNpLaEQplo4X16P3UzrUnAfNq8VAMG3nMwzvsQTiY6BHMk8BDizqTlLE9SbRHrwm5JUw3JozP95oQHsLPTLzJr0rnyY1DkmJykUUjjxshToQ21Wu1+lSuPvpAWCohRXnjZ4HE+vlzjz6ST8dEn/wmVecJryNsa0ZL8moiVdWyXV0Je0JXSXgawxPzdfIdI0Sx3L7Hsp3pc5FjH+uv9UmoMG729YR0c9xsNCc/97LnVratWIaQsjwfQvNM3Cw5By+55JJgnmaBfcsOjBDr4Tyv1Fa2XaZttDcPDe9qPfl6mTcxyc+xGSxxbLlzzEStYC9l/JjXw5S6zXvP95HQOZkIiIAIiIAIiIAIiEB1AhKZqvPRWhFoOgEeYOxNerDQI08Uz4MQifBJMMsojDIREAER6A4C5IRC6CTJNWKiDQPt1yDELgsT7Y4uqA0REAEREAEREAEREIE2J9DcV8otAuf2228vOzpZi3RP3ejlBBgKlgTteF+QkJ7kgCRBl8DUy08M7b4IdDMBErJbqK17qjHSIF5KeNxdc8013dwTNScCItAuBBjQhIFwLLy20KAVrbhfDKZjuWB9UIxs/yxfng/Ggmcp6y1sOLu64dN4C9POK6+80vC6G1Gh5Qb0+9RG1KU6REAEejaBXuHJ1LMPofZOBERABERABBpH4IUXXvCRERk9kPC5fFhd41pSTSIgAu1MADGEUS0Z3ZGRoi3HXjpcezvtF6N/MYqk5bws6T+jKDP68EILLeT7iEdnM18AkjZh3nnnDZYbMSyzzDIth5ARN+++++7ASNMyERABEahGYKxqK7VOBERABERABESgdxHAi4k/mQiIgAhUI4DYQD60iy++uMddM2w01vDQQw+FSy+91EOHq3HQOhEQAREQgVICEplKeWhOBERABERABERABERABESgCoH7778/2GiYXoI8bjYqblhllVV8/qmnngq33nqrDy5ho1WGDTbYINgoo76OXG/XXXedh9jZaJY+0ADJ+vGIWnzxxdMWGXSgf//+YbnllkuXXXbZZZ67Es8ijD4wYMF7773nA0usscYa/sm6cu1stNFGgQT+DHphI8a6x2YlzyQ8dgYMGBCmm266VGRiwIMrr7zSc2ji5Yn300QTTURzqVXrUyyEB9hdd93l9dJnvEYrmY3k6p5DlLvnnnt8/7qLEwNJcIxhSe5QmQiIgAgUJmAXTJkIiIAIiIAIiIAIiIAIiIAIFCJw2mmnJRYmxwjVyUorrZQcdthhvt0xxxyTmGiU9OvXLzFRJ7FRShMTURILRfP1I0aM8G1WX331xAYY8LIrr7xyYmJP2q4NgOJlTKBKl7G9he4mNiiKL7NReBMb2TKxkLbEBJDEBi1IJphggsTyK1Vsx3IqJZbzMrFRKBMbZTJZb731fHr++ef39mIfqWDHHXdM9t57b6+LOtlPC2VLTFxK1l577cRGmU3mmGOOxPJpehn+1eoTZSzHXcpnrbXWcgYbb7wxq5JnnnnG27FwOZ+3UYYTGwgm2W+//Xye/nYXJxO3vO2pppoqMZEwmWKKKRLzcPU/74z+iYAIiEAVAqHKOq0SAREQAREQAREQAREQAREQgQ4ELr/8chdFPvnkE19ng5i4+ILg9MMPP/gyS5rtApDlNvL5KDJZDqfk22+/TT799NPkoosuSmyQgcQ8jLyMJdxOJp54YheVPvzwQ1924YUX+rLvv/8+oR1En7/97W++jn/mzeRt77PPPr6sXDusWGeddVwoieLQl19+mcw999xeX1ZkmnHGGRMbOMjriiLTggsumNA+9uqrryYIMNtvv73PF+kTIhfC2P7775/yMe8kb/u+++4rEZnME8yFsCje0Uh3cfruu+8SG5QmGTx4cPLNN9/4/llopPcHoUkmAiIgArUI9IrR5Qq7damgCIiACIiACIiACIiACIhA3QRuvPFGD387+OCD0/C4aaedNgwaNChcddVV4Zdffknr3GqrrYJ5MvlolubVxEtvDyGjwB133BG23nprX08SbMy8esKqq64azAMpmNgTzNspbLrppr6ObT///PPQt29f//SFv/7LtkO5O++8M+ywww7BPK28hIlZYdddd81uEl566aVg4pYnM8+uIOyP9rFZZpnF+3jttdf6fJE+EfpGDqtDDz005UNy8SeeeCIQVhiNUbHXXXfdsNtuu4XDDz88Lg7dxYkE5O+//34wMSyYd5i3T54++iQTAREQgSIEJDIVoaQyIiACIiACIiACIiACIiACFQkgziC+mFdSSRnEIfMACu+++266fKaZZkqnLRTL8zEhriBEka+I/E7kHiI3knlFBdaZF5JvQw4nRCCEKPIZWahcGDhwoOcO+vnnn9N6mci2gzD19ddfh5lnnrmkzGyzzVYyT74mxJ8oKMWVFtYXJ/2TuslXRJLwIn0iVxVCGOJa1sgxNemkk6aLjjrqKBd3Hn744RJhrrs4Wdheun9pp2wizym7TtMiIAIikCUgkSlLQ9MiIAIiIAIiIAIiIAIiIAJ1EyAJNiJOObPcQsHCy9JVeQHH8hN5Mu6RI0cGC2ELSy21VFhxxRVdcLJQMhepVlttNd+eUd+WXHLJYKF4YaeddvLk3xZ250m60wZ+nci2Y/mhguV16tDHH3/8sWQzEpnnBSUKsA9Ze/HFF12IQuQq0ie2txDBbBU+jYiWtT//+c/h+uuvD3g+nXLKKdlVoTs4TT755N6mhcqVtJ3nVLJSMyIgAiKQISCRKQNDkyIgAiIgAiIgAiIgAiIgAvUTsNxGwXIVufiT3ZpR0Rg9Li/SZMsgnrz++uvh3HPPDYsuuqiP2obIxEhsZ511lo8yF719GGUObyC8mwh9m2eeedybiO3znkzZNth+1lln9VHpsssfffTRdBaPq39ZiF4cKS9dYROWdyk7GxCj2Ge8mIr0CQaWv8oZxYos/5GH7p144olxkQtoiGyWfDwcdNBB4eWXX07XdQenhRde2NtjpLysZTlll2taBERABPIEJDLliWheBERABERABERABERABESgLgLbbbddwAuGXEmIQ3g1nX766eHKK68Mu+yyS9W6BgwYEGy0tmAJvt2DicI2ep3XR96jGCrH8sUWWyxYkvAwfPhwz3GER5GN+OYCU977hvJZI8/Q+eefHyyhdrDk3+4plBV48JqypNeB/uTNRnkLrGe/Tj311EC7hxxyiBcr0qcNN9zQQ/UsoXawUfJcjLNE5b79kCFD8s2FY4891vefsMCYz6o7OJGvyhK1hz333NOPx5tvvhl23333gFiYtauvvtpDFvE8k4mACIhAloBEpiwNTYuACIiACIiACIiACIiACNRNgCTaJNYeNWqUizTMn3TSSeGMM84I2267bc368NIhMfYKK6zgZfEQWm655TzEjXXREGsQshBfSEyN5xN5n0jg/dhjj3kS8Vg2/7nNNtuEoUOHhnPOOcfD64YNG+YJrmM58jGVC5VjPV5Fa665ZmC/mMbDKopfRfpE6J6NGuceXSTx7t+/f7jmmmtc9KLOvLHs7LPP9lA8OEbrDk42wp+zRxy0kfbCAw88EHbeeefYBf+0EfyCjXgX8MaSiYAIiECWwBgMP5ddoGkREAEREAEREAEREAEREAER6CwBRnvDq4jR5Zpl5DIimfgMM8zgQlQ97fD489Zbb4Xpp5++ZFvC/Qirm3LKKctWRzgenj202adPnw5livYJPuSRqlRPh4q7sKBon8o1QfggIX7NPI7l2tUyERCB9iYgkam9j596LwIiIAIiIAIiIAIiIAIiIAIiIAIiIAItQUDhci1xGNQJERABERABERABERABERABERABERABEWhvAhKZ2vv4qfciIAIiIAIiIAIiIAIiIAIiIAIiIAIi0BIEJDK1xGFQJ0RABERABERABERABERABERABERABESgvQlIZGrv46fei4AIiIAIiIAIiIAIiIAIiIAIiIAIiEBLEJDI1BKHQZ0QAREQAREQAREQAREQAREQAREQAREQgfYmIJGpvY+fei8CIiACIiACIiACIiACIiACIiACIiACLUFAIlNLHAZ1QgREQAREQAREQAREQAREQAREQAREQATam4BEpvY+fuq9CIiACIiACIiACIiACIiACIiACIiACLQEAYlMLXEY1AkREAEREAEREAEREAEREAEREAEREAERaG8CEpna+/ip9yIgAiIgAiIgAiIgAiIgAiIgAiIgAiLQEgQkMrXEYVAnREAEREAEREAEREAEREAEREAEREAERKC9CUhkau/jp96LgAiIgAiIgAiIgAiIgAiIgAiIgAiIQEsQkMjUEodBnRABERABERABERABERABERABERABERCB9iYgkam9j596LwIiIAIiIAIiIAIiIAIiIAIiIAIiIAItQUAiU0scBnVCBERABERABERABERABERABERABERABNqbgESm9j5+6r0IiIAIiIAIiIAIiIAIiIAIiIAIiIAItAQBiUwtcRjUCREQAREQAREQAREQAREQAREQAREQARFobwISmdr7+Kn3IiACIiACIiACIiACIiACIiACIiACItASBCQytcRhUCdEQAREQAREQAREQAREQAREQAREQAREoL0JSGRq7+On3ouACIiACIiACIiACIiACIiACIiACIhASxD4P9UJI2/Ixb1iAAAAAElFTkSuQmCC", - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.display import Image, SVG\n", - "Image(filename='../../../docs/source/_figures/remote_2.png')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As seen here, the GraphStore is used to store the neighbor relations between the nodes of the graph, whereas the FeatureStore is used to store the node and edge features in the graph." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's start by loading in a knowledge graph dataset for the sake of our experiment:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/zaristei/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from torch_geometric.data import LargeGraphIndexer\n", - "from torch_geometric.datasets import UpdatedWebQSPDataset\n", - "from itertools import chain" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Limiting to 10 questions for the sake of compute, but can be increased if necessary\n", - "ds = UpdatedWebQSPDataset(root='demo', limit=10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's set up our set of questions and graph triplets:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['what is the name of justin bieber brother',\n", - " 'what character did natalie portman play in star wars',\n", - " 'what country is the grand bahama island in',\n", - " 'what kind of money to take to bahamas',\n", - " 'what character did john noble play in lord of the rings',\n", - " 'who does joakim noah play for',\n", - " 'where are the nfl redskins from',\n", - " 'where did saki live',\n", - " 'who did draco malloy end up marrying',\n", - " 'which countries border the us']" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "questions = ds.raw_dataset['question']\n", - "questions" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['P!nk', 'freebase.valuenotation.is_reviewed', 'Gender'],\n", - " ['1Club.FM: Power', 'broadcast.content.artist', 'P!nk'],\n", - " ['Somebody to Love', 'music.recording.contributions', 'm.0rqp4h0'],\n", - " ['Rudolph Valentino', 'freebase.valuenotation.is_reviewed', 'Place of birth'],\n", - " ['Ice Cube', 'broadcast.artist.content', '.977 The Hits Channel'],\n", - " ['Colbie Caillat', 'broadcast.artist.content', 'Hot Wired Radio'],\n", - " ['Stephen Melton', 'people.person.nationality', 'United States of America'],\n", - " ['Record producer',\n", - " 'music.performance_role.regular_performances',\n", - " 'm.012m1vf1'],\n", - " ['Justin Bieber', 'award.award_winner.awards_won', 'm.0yrkc0l'],\n", - " ['1.FM Top 40', 'broadcast.content.artist', 'Geri Halliwell']]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.raw_dataset[:10]['graph'][0][:10]" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "all_triplets = chain.from_iterable((row['graph'] for row in ds.raw_dataset))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With these questions and triplets, we want to:\n", - "1. Consolidate all the relations in these triplets into a Knowledge Graph\n", - "2. Create a FeatureStore that encodes all the nodes and edges in the knowledge graph\n", - "3. Create a GraphStore that encodes all the edge indices in the knowledge graph" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "from torch_geometric.nn.nlp import SentenceTransformer\n", - "from torch_geometric.datasets.web_qsp_dataset import preprocess_triplet" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.append('..')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In order to create a remote backend, we need to define a FeatureStore and GraphStore locally, as well as a method for initializing its state from triplets:" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "from profiling_utils import create_remote_backend_from_triplets, RemoteGraphBackendLoader\n", - "\n", - "# We define this GraphStore to sample the neighbors of a node locally.\n", - "# Ideally for a real remote backend, this interface would be replaced with an API to a Graph DB, such as Neo4j.\n", - "from rag_graph_store import NeighborSamplingRAGGraphStore\n", - "\n", - "# We define this FeatureStore to encode the nodes and edges locally, and perform appoximate KNN when indexing.\n", - "# Ideally for a real remote backend, this interface would be replaced with an API to a vector DB, such as Pinecone.\n", - "from rag_feature_store import SentenceTransformerFeatureStore" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "model = SentenceTransformer(model_name=\"sentence-transformers/all-roberta-large-v1\").to(device)\n", - "\n", - "backend_loader: RemoteGraphBackendLoader = create_remote_backend_from_triplets(\n", - " triplets=all_triplets, # All the triplets to insert into the backend\n", - " node_embedding_model=model, # Embedding model to process triplets with\n", - " node_method_to_call=\"encode\", # This method will encode the nodes/edges with 'model.encode' in this case.\n", - " path=\"backend\", # Save path\n", - " pre_transform=preprocess_triplet, # Preprocessing function to apply to triplets before invoking embedding model.\n", - " node_method_kwargs={\"batch_size\": 256}, # Keyword arguments to pass to the node_method_to_call.\n", - " graph_db=NeighborSamplingRAGGraphStore, # Graph Store to use\n", - " feature_db=SentenceTransformerFeatureStore # Feature Store to use\n", - " ) \n", - "# This loader saves a copy of the processed data locally to be transformed into a graphstore and featurestore when load() is called.\n", - "feature_store, graph_store = backend_loader.load()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we have initialized our remote backends, we can now retrieve from them using a Loader to query the backends, as shown in this diagram:" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAAGRCAYAAADmVIAxAAAAAXNSR0IArs4c6QAAAFBlWElmTU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAACDaADAAQAAAABAAABkQAAAABhaUSHAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgoZXuEHAABAAElEQVR4AeydBXzV1f+HtwGiqCh2oAwURSUUEwNmtwIqtmJ3d2P/zL/dAXaD3Yrd3ckEW0FURCW2//PcfQ9+vd7FhbsxtvN+vZ578nvic/Le3W1FRVHRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtEC0QLRAtECTdkCxU25c7Fv0QLRAk3XAm3atPmZ3s3ddHsYexYt0GAWGD1+/Ph5Gqy2WFG0QLRAtEBDW4BLQ2VD1xnrixZoihbIZy2VNEUDxD5FC0QLRAtEC0QLRAsU3gItayoyfvxXk3ViWrRAXhaIH//lZa6YOVogWqAxWqDGSwMNnpufc8TvPTTGkYttmqEskM/HfzNUx2JjowWiBZqVBeKPJ5rVcMfORgtEC0QLRAtEC0y9BeKlYeptF5+MFogWiBaIFogWaFYWiJeGZjXcsbPRAtEC0QLRAtECU2+BeGmYetvFJ6MFogWiBaIFogWalQXipaFZDXfsbLRAtEC0QLRAtMDUWyBeGqbedvHJaIFogWiBaIFogWZlgXhpaFbDHTsbLRAtEC0QLRAtMPUWiJeGqbddfDJaIFpgxrNAN5rcD9aDdvXU/DkotwwWqmP5rci3MmyTuC3q+FxDZOtBJVvASlDov9mzBmXOBNNbq9OA1jAnrDAdGrNaUn99VN2ZQhetj4Jzlhn/IE1Os8TIaIG8LRDXUt4mq/WBPG26CAW+Av6/isCf+PeEQstDyDr2qkPBtuvTJH9o10eEF06eXRB3CCydhBvKKaWiV8G23ASfwOswPxRK4ygo9LNQZU5NOb/xkONQBm/B1Go/Htx4Kh4ewzPWXx+6mEKPqq3gfNZS/KThH2t60yyFWf6JajS+9rRk5kbTmtiQaIEZywK+Q74PesKR0AX6w09wWRLGmS46gloXB9/Nd4TjwPYdD2o32AnsQ0PJc2EoPA5LwQ5gm76Ey6GpajgdW24aOrcpz3qONF/lcfvwI51wS067TryplR/ReVPvNLUF1PE5Lwk3w99g2ycm4Ta4ygXkuxEXdkPKS8L58AuMgN/hCjC+UBpLQR0KVdg0lDOaZx1n36G9Nw3llPHsalPx/I8844Zdb8pjLdVbG5pawXnYdC367to+L8sGqxDeGuZL4j/GvRKcg77jdA9yDTo/foWXwY/p1f/gDfBd3HfgO3EPDeU8tj4PWJ9xDV8Huf5s/9XEm/cg8ONx69wMusOKMAZM/xrWBC8Ph8FH4Pp9GDzY1dowAmybbT4ZZoPrwXI+g0OgNmmvH8Afm6Tlj1tMUwvA3fAFWNet0AbUO3Ao2OaRUN2nOX7ScBGUw/sQ7Ic3My7GjQLrmRey1ZuIG+FF+AC03QAIz92DP4ztqfjPhTdAG50H5le/ge/0nQ8PgpoZQtucFwcaiWzHbRD6fRf+tnAAjAftZhu0nfWVg5etQeDYqZ7wEnwDV4DjmOuTBufRC6AdbfMxoLzY2IYbwGft08qgZgXH27J99hE4CmpUHmupxnKK8ihoBUqqhOfgpBRb459aHc6Dltl9aguo43PHJ/Wcj7szOBjWewaoQ8HwsgYaUHdQlxPYSap0X4GLDRRITrgOBSprWooZzcOdYA5YcRoKeoBnXbD5yk2vY74P5ZM/j7WUT7HNOm8eNj0CQ7mGN00M5iZflqIUv/oTzOdG7jMbgQfBCbAteHFwE1Ye9hXwJOwG/ojB5xeFcGnwwHQPvAXS9ROcom74XIemTwI3+r3BA82yngLTLoXOcGASvhv3YPgevoM5YRMwr/P5bFgbroe/wDKNM70/1CQvMI/WlIG0W+GsJM/suF60tJH6Dc4D+6At/oZZIFvjiLgEPEyXh9HgxcQ9wEPPPUFp/wczvn+/rEdQm9nvZWAF+BYWA3U8PJTxVX2i9An++aANPAP7gLK9Htpl8Bao0+BemBnmhXLoBdfCRWCbLec12A2UcyPY9hT894P9ng2Gw57QEr6CsE9pa8fE+tMyn2O7ThK5FK59nQdsh3NvU1Anw5MZX1HRObh3gc8vCT/BUVCj8lhLNZYzNZeGM6spcW7iPZSfhv+D+SFI4w0D08yzICwOb4DGNG1Z2AmuBwdRbQ+GHbjuiX9L3Idhd1AOkgN5B/SGXLqOSOtxIJTlnwQ7gpPvdTDdCdQT1Gpg3Q7UIGgLqhsYbzucrNav9gDbcSf0gdq0DBnGw1xZGXsQPiKJc1IcBvbXsg+FElCXgv0dmuDCzaWxRO4KLsgboAsEOUmHwBNwNFhftsxzMDhuN8NMYBnp51oRVo6XY6197oNtIMjNohN0htNCJK55HP/bIdjNjehAsM2PggvCuI1hFAyHMlD9wYVrfaYHzYXnXHgc7P+P0BHqTQVblPXWwhmv4Dxsegy9q4QwB1ZNwsaJm676E97K+P55aY3X9X4k/AzuS+pq8NlSUM5tw65115v+Y0GtAYb3NZBD8xNn+a61v8C8F4M6Hgy7J6gP4CtwzquBYLruJok/1FtMeBy8CmUJXnxug5p0KIkP1ZQhSZsDd3lwbb8He4H6DRbL+Kpe/sBZIBUOXtuWjn+A8EA4G9xvyxIct0nQBtJaj4D2CDoLj2u9LGEj3MkwK1wGh0PQtnjcO5XtXQTKIIy//UmfGfMS1p7K/b4nWMabcAioR6B/xldU9DGuY1qWcAau6SvCCAgqwfMLWH+23DvnAufT/jAeOkEv+AKCnJ/vJIFPcA0HXY7nqBCozs1jLRXZ4ELKzp2UQuO2gMdhJ9DA/eBp0CBrghPYSW+ahrkEXKhzgvLm6WRZBQaCz6mVYSCYt33ivx7XOi3PhXMl/ACW9SRYRrasvwJehI/gTLgfbgTLbgcqtGN9/M+AA+MmYj32ZyZYGAbCYHBBWe7RcBX8CLbtCVgVapLp2mNMViYnxjlJnIer9jsMnJAe3puCcjK7QP4Hz4KL0fblkuNyDLwET4Ft1J72ybYeBD3AccnWAkRY90QwrwtreOI/ELc7hOccr0vhTrDtp0BfSGt+ApsnEbvgHg9ng+PoxaELGGc/XZCWsQcMABf51zAcPoHtwXocT212PriJqHuhJRwBjuM8ENV0LfBB0rUVE/d9XNeOcztbI1MRzlnDQ6AU3LTdW9IakwRc32r2KifzGtJcH6q4yvnX64GENgPn+TqwMJTDjpBLHiLfweQkcVTizp24OqEPM+P3wOwAJyW8ievzNeljEpfKkWFe4o4D9+C9wP3yROgMv0K6f9oqyLam00K8rs8F6dd+7WAO6JOwAu6p0AKy9UsqwufaQnjO8XaPKAHlfh30G570WIX44HpmjA0B3J+gEnYG7TMIlgbz5OqbbekEoS1/478bbJ/9DPKMSPchxFum58YbsB/Yh0kQ6voDf5BlhHjtli4/3eeQv/7cPG4fDqoGzaY9cWsm8Qfgqk3BfOuDk3o50JBrwFegkZQHmfm6G0CXgeHZDaCLwLCD42Gg/3wI+h7P80nAST4GrkvC2U4ZEXeCxrYcByG096gkbllc9Sw4yE4qdQj4zNawQeK/ADfoWzwvJgHbMRoGJ+HqHMt8pLrEJH4Z3LBRLI7/aXByKfsa2mt4HLgZZWssEU78oCfwbAfHwr1QmtAT10nveKXl2H6dinDM7oPShOVww3OO1xkQtAueYUlAm7jAVgcPf+XYbZ7xVb2shrMguJnNVxWVeUY7HZGEH8AdkPhfwXUMSxMcR9vmBmd9YQNqjf936Aj1pjzWUr21oakVnIdNnbcj4Fdwr1ALQNhTTs7EVH3S4D4QNBiPa7stOF9GwuugrgbTXC/qRDBs+c5j/R6sahUwHNancUHOc9dI7yTCdfAjlCdhD2mf7ZGEne/jYbEkfCmu6e6fmyR+96Kgcjzvg+23H6fAqqB6QXqfyETyor2+gW1DROK6fl9N/K6hPolfx/W2TxL+Dde1GpQdDvHuSx7sqgQ+hDXB9fwgBC2Ex7rNk9Z6BLRf0OF4HgoBXNsQnnOsz0ulnYD/yiRs+xaBMngL1NOwfcZX9XIxzsYwCjasisq8mu+wJPww7haJ33btmfh1fHYALAzuN7ODmhP+ButPa3kC7s+zJJFz406CxaEXvAtB6fBTRO4QEnCHgntfjcpjLeX8yLnGwmtJ/B/px2TlWSsJX4QrQW7STubTYW1wwrSB7IlBVE7lyvdektODYP4EF1RQp+BJuavhnwBbgWU6aW8FJ9XFkC0X62fggKqwiVh2mHChHa2IWzChtnaQbYpG4lt0Sugfj4vZ9g4HN4ArwHq143xQDEEu6iD7Z99yybqCXBBupk7g7nA9BL2Ixzr/ChGJ+20q7HPdINdzZvvSl0Rf47oZVCcXl+0JeiHxGO8GsAR8BPNCut8EM2rP607QvyqYeXXcfP47mJyJqVqw6T4k0dFpQhZwzrq+vQinDyO76Jy4X08OvU3cznAfzApho8c7RefjOxaWBg/Ox2AVqKuOJuOj8AyMB/dA5+ZhoFwnynL3But6Et6BMeCauw6eAy8NKr3XnEB4CLgnuR+5LqxP3Qmuyd4GUtJerp27YH34AtwP+sA6oNyvdwX3HQ9R9zkPwHx1AQ+4X1iP63A4vAv7wOXwGtjv56ECatJ1JO4LV8CrsBe8COG53fB7cZwMll8G1eksEq4F9xf3PW07COy3tpkN1obFIPTbPde2OmanwE1g2gQ4BraDb+AGeCBxt8f1MpCtUUnEAbj6dwPbHurCm1PWezPMAx3AMXsZCqaWBSup+oLcoJWdvwQ0shPNSX8uOOE6w+fwAYQBxptROBCCYV24v8NCVcn/ev07Cek6gB4qa0AL2ADehGxdRkRHWAoc0GfhF3ARpBXaMYLIZSC0o2eSyYUVFNoxkYifwI1pNaipHSRP0ZP4XEgrwytTYqs+zTiNcCk4Ic+Ea0C5GYY2ZiLq+LII+bST0g4PgX17AXYANQtsDD8byFJlKvw9fhepC0FlP9e+Kjrzal0jU+Fs7w9EmD+M2UD85XAxXA0XgboT0v0Ofp8/B+4AtSS0A8fDuePcn5S4zseopm2B1+ne4uA8XgYmw0swPPHjZH7U9bWeRBfifgddwPUwBrqB8zpoTTybwzhwzTqnPoWBEDZr9wbDr0C2nifC8rcED6c/wbXsXqhugZnBOerB/zmYfxvwAHkNXLPKPXUgeGAG3YjH9b0B2LZh8DGoQ8B255J7UFfoD3PBU7A3uK+qzWBnsC1XwVkQ9sx98I+FoOxwiN8Lj+Ni3x+HweB+Yh3Lw3bg/jQIQh/xTtH7+E6fEqoaH/dj9x+f8wDVlkHn4XFvnw16wVegbJ9jOxFOBPUIOFfkV1gBbNcA8NKwFNwE50BHUEeCdVvOY+Dc2BRmhrUgjOn++C1H2x0D7cH60/qRwKrQFxaGXWEx+A3swwkQ5JwI4eH4N4B+8C6sB+E8wlvPyuMjCw3qYJ+Zo0mtiNNYGuWMxG+n5wcH3OfOB2+HXhg+BeVAmvYo9E6FnTxXgcYzvR1slPhdSEGn4jF9CNye+B3sbDnI5nNCPA0jkvDZuMqJbbqToA+4QUyGD8EF7YA48e2ng2Xe7SDoZDzG3QC3Jf6BuKocXtSTQ9brxNEOZXAEOHldYMr6jSuF3WA8HAlKW7togrLDId6FfS84eQ+DEdAGOoFjtDssDteC45AtF8Urqcjw3B7E+dw1oN3URWB/1ocysK51QWl7n10d3BiV/X8HVoZ14BtYErT1iVAKO8I4GATKcb4SSmFP+BT6gPPTBbQvKNt0KXQF2zUJOkK9KY+1VG9taGoFT2ebXo09XdezNTW7NtH+XEa/Dm+ifZvmbuWzllrUVFurVq0GTZw48eSa8iRpM+HOCc+Ah1laXgTuhrbg5u2hsDuUw6vQGnqCB8RgaAlPgLdhLxY+9zTcD7bXw8VLiIfDn3Af+Mws4HMeLsq2/ASWrU6BmzO+f7/YntfAemaFr+FsOA/cFL6E+cD04fB44i6I6w3wFvCAsy22IbTDctSz4GFpO4rBdtwIqj1Yv3my9QYR78LGsBm0gUPBG7B6DraCrcE+XwLW7aG6KHjI2yaVHa6KrYofSuA4sO8D4WfwwuDzXn4s37E4BP6GtKyvFWhr5XO2b1vYBj6Bg8HnNgTLWRnK4FQwr7J9HuRexmaGp8D+T4D9wMP9MHgb7HdfsG0/wf/B7PAKfAWrw1hwrEfD3rAGDIGrQA2DXrAbPA/vw3AYD/WiPNZSvdTfFAudzjb1Auu+dxNMaor2bWJ9WoL+uE96dkRlWaBgaymf20dWG2IwWiDbAhcRcXR2ZHMJx7VU+JGONi28TWOJzdMC+aylkuZpotjraIFogWiBaIFogWiBfC3QMt8HYv5ogam0wIFT+Vx8LFogWiBaIFqgkVggftLQSAYiNiNaIFqg0VvA7zEsnrTyKNwtEn918UlyvTmPUvIctZTu96C2rSVPdcnLkRC+B1RdHr/r5PeL1AMwf8b375eVCPrF43zkd5vWzueBesjbgzKvScr1u1G7TkMd6/NseJN+K/4wj6ahyOnzaLw0TB+7x1qjBaIFZjwLXEeTOybNfgH3o8RfXXySXG9Ob0r2i8g16SUS/bL11Og7Hrq/lgfT5fslZA/7bLUjomd2ZC3h7UnfvZY89Z08JxX45X3lF8zfyvjyf3GM/NJ3i+TRu3DHJP4Zzgk3n8bScG91foxdX99iX5SyR8JSMABOBlVdfFVq/bwOpNhv4bEaiu9M2s5wfA15ako6n8Tz4JtqMnnb3QX87Ykd4FfItUn4JcbT4Qeoq9qT0f5V1PWBmC9aoJ4ssAjletjPB/4mzrtwDwSth2fVJOBvaj2T+D24PocNwIPX5837GQQtjydXvOnur74L7wSuhRvhT1gYFgMPWOv9GobARFC+w98UJsAd8CWomWAgLAT3Qb7yEwcPedvkurwF3A+tywPRdijbK/5WUZBt3Qms2/3kJrAv2bJf4VehB+P3N5my1Z2IzWESeICm7Ukw8+nJariO2TrwBMwLltsWXoMHIZc6ELkVzA5eem6A8bAS2N4ymBvuhXAJML993QL+guthNNSkLUl0Lrknmt/nWoM2tt2G74ZPYUNQtl+7pTUXAe3qBeVtGAbKeWV7V4FF4VV4CKa7SqZ7C/7dAI2n4etD21PoaTkKri4+R9aCRq1KaUsXtMRpK2xlHncx59I2RLpY66pWZPwIWtT1gZgvWqAeLbAsZXtIeDh6YTgOjgR1GHghLoex4GHcC9RVcC7MCS0hrd0IrJuOSPzpeA+N7WAErAlPgWujG9wJA2Ek7A9hb9oI/1DwkuHB8yz4Jkd5wHoAlcOF4CWiNu1MBstUl4CXk9HQCSzbNdobDoKgY/BoM+sdBOpScM/6EOz3DaDcszfO+KpeLsAZBZ3hOchu4zrEPQA/wjjQJtXtOyRlNB+vr8NCYNlnwKmQLS+Hr0AxfAnu7ReD8jC3rlmhHLx02B91HtwKn8L88AJkf2KyNXH9QWl77fUJOFccZ+X4aBvr9jLwMrSBbJ1LxKLQDuyXl4yv4HgwTXmReRRM+xluhDCOeBup8vk1DLrg5FgDHJylIWhJPPOAE2NLmBfSWoZAX5gDJoCGzNbqRGj8zWGlrMRWhFeD7aBrKs16F4ANkvjzcR0E87gJeJNzAuWKJzoj22SdToTWmZiqF+szbPx6YBvSWpGAfV0wHYl/brCvS8BVcDDUJOu3LLU4OKG1l2XbN1UKlpeW9poFekGYtI7BprAmOFaqLYTyXVxupj3BPs8GQT/i6RwCuNpuK1g4FZf2rkKgEtaGYBsn/zawHDQ75bmWmp19pqbDedjUef8JFCf1uD/9kIRNc69QprtH7GEA/QWrZnxVLy/grJuEfXd5UOLPFb8sad9DWGuW/Q70A/ckLxJBruenk8CruANCAq5r8kroAj9BK1ALwWRwXdekK0g8MskwGte9QZXAeLAc7fE1GDczeEgtAGvCG6BeBA8190wJ+8Zl+I8GNRbS9rIv7iXrw0ugngPfcQcdjmdwCKTc3fDfmoQt/+ZUmpeDv8D9K63FCLi/BrlHeYlQ/wfaMWh/PLcngZG464UEXMfCMegDb4M6B06FOUC7eSlQLcExcpz3hdag3H9/Bfdtx6wSQtoI/M4P5889EOT+br/mhf/BEAhyfz4pBArt5rGWMpOkEPXbyfdhP+gFT8OBoI6Ch+AM2B4+hVJQxt0FDpi3QCdtLj1ApOm94UIIk6kd/ndB468Mj8ERoJyMbgCDwDasAk6qtcGD0Qnk87niic78LOsTXCehk8F6wiE5DP/DsBOcDrZNtYB74Txwcb4MbkpqRfgQNoLrYH2oTd3IYF7lJL8fzoetwbYtDU7KuyHIOO0zAe6EjrAMvAVlsDfYlzbQFQZD0B54TgP7/DH4bFqOj2VeBKvBC7AFZGvDJEL7uAkdAM9Cb7gG7oDqxpqkqGiBglvgHUp041YfgXuWB8B74P7xBjjnu0MxBH0ePHm6rkvXu+tQWffbYLz6usrJvI7j1b1DdYH/gxEJtq1zgm2ZCOpb+Cnjy+9lVJK9AvdPaAm28xtYCzaHF+F7SMv9tB+MAfe/uSCX3FuCLHexEEhc+3cWhP65X9u/mqTNHL8g+/ArLBoiErc8iXsc9ws4BdL7jPYPcg6EsTDurZCAm52WSsqcXdpdO6hJcDo4zs6lG+EDeBm8JKTrJ/gvZffrB1K1e2mSa2Ti6vwBLVLh6eZ1whRCHkpXwzlJYQ7AAPBwURpxl4yv6gLhJeFB2A86ggOwArwG1cmJNhRagYt75cTvIJ0B6hXYA0I7PsW/Fai9YVW4ENYC9TXcANnxpp0LJ8JVBpDPDQLLV5eCh7WH7y+wIKwJ80MvcJO4CWyzh/1pcAJYnoPvJpWvRvFAOKRvw78RnA+zw3Kg3b3F26fJELQ2nlfhsCRiX9xZE3/acVH1TSJs7zGwZxLW6QeLw/LgpnMtuEDto+Egx0Pb7Q6zwZnQDdwoHD/baVnaLypaoCEs0C5Vybz4/4bf4Tm4H1xLbtr6iyEovY5CXF3csWSaJyuj4XDYuD/k0q9EbgMvJIlz47YE153+IPcQ11a+qq7e6yloO5gPXNfZ+pmIlWAB2ALuS/w4/5JtHJfE2N/R/0qtOuzdU55K4r18zJSVJzuYbUvzt4VfsjJ6nnjOyHuwCRwPQXMGD65zwD4FmfZTEjDt7ZCQ5f5G2LwlEPa8XfF7SXgEdoQn4HdwLGvSWBLTc8QxdZ6GflU3VjWVWe9pdrwQGk4hL4EHuweBB4YTPSg9AC5MB70neKMLi+h1/BqxOnk4qYng5WB5eB6eBg+lu+BUSNfrQTm1svxHUw8/jH/FVNiDT40HJ5K3ylXAyXwduAj3h4VhQbC8Z0G5EQ3Xk6dy2dGJOwR2AMdzexgMaQ0j0AO+Adv2KYQFgneKgo2NsK22OS371wbcVOzfoeACWxSqU1cSRsKIJIPj56JaIQlHJ1qgISzQm0r6gJfWQeB+4Tr0kPSAdl9aFdaC6g4xLxpLQPaFO1e8Zbo2dga1Prh+PFhq0p0kngjuI3PAveAh6J7nobITqMMg3Q7rmduEqZRvQtaDZeGBHGXcQdxAcA9x/dpn7ZetI4iwnWuDl4yHIC37dzzMCbOD58VekC3LXxB8EzYUdoTu4P4+CNx/bUtajqX7zDswG7j/psdSO7YH7XQI2OegY/C0AueA8+R+yCXLl4OTxA1xB4HtqoTn4Xc4EOyf9U8CbdUNtE2Q+/LW4Fno3n0cfAmfQ03SrmvUlKE+02xoITSQQm6Cz+A0OAGKIUiDBWlY5aRID6hx6WcMp5U29swk/AXbgpPZQTwDjoa0JqQDefod6HT7vBT8mSqjIuUPXvv5EpycMAi3E/wEtiVdXk19JWtO5bKjGYeAk8+F+hU4DmmNJNAZtoJR4MLtD9myj0HZ/TXe+r2Ihf7pLgbZi5eoKfKSkO63CZbt+EVFCzSUBT6koiPhUwgHinXvBefCS7ArHAseAOpZcP4Guc48LDaFj+FrULni3Ss8UHYG96cToC+4VsaAh15QOmz9tvVNeAOeg3PA/WgTsI2WtzB46IT2eeiVQrbsr2tevQDpdZcOjyXNflwAoUzjbIPyshLqvhn/9uCbpc9gJKjnwcvXJ3AmbAHufb+A/VEngf7X4G14HU6HbA0nwn3+MrCdh8NNYH2dwLKzdTERvqmxzGFwBXwLrUB9AF7CXk7cwbhBnkfa/ULYBr4D+29Z6gvQ7sq61wHDR4Hz4V04F54F2zs7XAB+klAJ9sOzqgNY/zjQtvvBtWD5PcCylJeHcj2J0uEViVsjJDQqN48vR1xNwzWYKoZb4GkD6DrYP+OregnhtgRHQ/ckbQNcjdsuCacdB2/PJKI97o9QCk4SCboej4tf2aaDMr6qlz1wHDS1FoTJUF38UPI48VULuAv+ZwC5EEr1JArhzQiPgDmT+PVwnYg+PxjOBuWEcvIfbAA5sebK+P79sjrB95IoJ6AbSlB2WHu/BbuGDLhuasvAYXAJBN2Jx7pXBReS0o5uULa1BMxzEijt3RnWB8ucG1QfsB+tDKRkGZUwG8wCjrN5lX0dBSFsXJNXHmupyduiUB3Mw6ZuxE8Wqt5GWo6H3QKNtG2NoVn/RyOOq6YhI4lfqpq0ZhGdx1qacqOeVsNcTwF3Q1eYEzy4PfC8QFQnb6m7wSPgLWoy/ArVaWcSdoFF4XgohyFwHywBc8AL0BE8tLLl7fYi8DLgARlUXfxBZBgGG0EbeB9Og5pkW7wBfgTe8G2LfbRvR8IDYH2W50EcdD6eMeBBPrVyDC4FD/ts3USEm6YXpUqYANoue6H8Rdzb4CXAMTkb0nqUgM950fgYFoO9YCKkZX9fgXLoBTuAF8lyKIXL4RmIihaIFph2CxRThHvV99NeVCwhWmAaLJDP7YNqWkNp4uLUWR6gHWrJPZb0hRN895rWTARKYeZ0ZDV+65JsVRdvPuudL/uBWsJtSS8F37Fny0vPrFmRpYRPyoqrj+AiFLpgLQV7+aotj5+UlEKu/hE9RX6qENQSTyn4bLNTnmup2dlnajqch039tGv+qakjPtNkLDAXPXFvy6X2RGZ/WporX5ONy2Mt1WyDghVUczV1SQ2XhrrknRHzbEmj40eLM+LI1bHNjWgt1bHFjT9btGnjH6PYwhnDAvmsJd/9zQjal0b+MiM0dCrbeNdUPhcfixaIFogWiBaIFmgwC8wolwZ/Hh4VLRAtEC0wNRbwY+m+WQ/+Tfg5+CYrvlDB1SjINzp+Eboh1Z3K+sE78B5sC6PAtvg9ox8gl7TR+nBHrsQ6xnme+J0pv9M0vVWI/tRXH5am4Hbgd/C02V7gj3Ifh2IwvtGqtp9JN9qGx4ZFC0QLRAvU0QJ+L+laKE3ogrsdfAKrQH1oMwrtVR8F11Lmo6T783u/1OybLfs6DvaBmr475qF1OEyLXubheaelgAI+uxBlXVDA8gpZlPPC+aH2BD9JnwjLQojHOwMqn59zFLh7fnEp/IzfX+9rW6Dy/RJifXy60oJyS7OwrvqUG8Oc9VlBNWX7RdS1oTv45dcyWBEcL8etOmmj+rZJqLszHttou7IvxtqtNAdumuFLtbnSfW6qNR3X0lS3ubE/mIdNfWf3d47++Fs8N2bFe8HIdfD5Bd5pmb/uO34ROagVnlLI/lK06R7u4deaDedSrueN851++FL4t/iXyPUwce6ppZC9PoiaIr8A7uGbS7YvO+134sK+nf1MaG+uL0IvSOZScI/Ilusy/SVWxyCXzcyT/sL6UoTtf1qmO761yX21FHK1xzLS7SGYkW0qBfeQbJnfPubSGUSeniuBOMemFAp1/lFUbuWxlnIXEGILVlAosO7upmR9Jsl+Ne5hdX/0XzldPLelYsbg75gKF8rrZuBitc1PJ3yHOwI6Q33IW/SJ9VFwLWU+S/qL4O34ZvBj0BPgedgIqpOLdVx1iXWMP598nWrIa5pj8CXcCu/Dx7AYBN2D5yMI4xTcPYlbJRX/Nf5RqfC2+Kda03EtTXWbG/uDedi0ukvD9fTxqqSfbuqvJjg/bgcPOrUzlMOj8AT8DB5+/hjgSQgybB51LRyS8VX9hcBH8P8Ba8IGMBKce84x555y/lr3s/AJ3ALFkK31iUg/v3eSwborYThY9t/wMqwDrtPeoI4Gn3etfAE9oAu4Z6k2cD+8B2+A7QmX5rfx3wiW9xU8ACVwFkyCF8CDPa21CbgmbZPPHAfK31p4E2zjK/A9dAf1Ori/OCY/wf/gMbB87d8LlO27D56DT+EKUOlLw8yEh8IHYLkvgZeRXDqDyI9hODg2q4KyHdrEfc56roKgvfGkx8PxUbPAveCY2s/HwcuF88L5sS1oj3I4F0I83sy/BfgM1/5qu6Og3pTHWqq5DQUrqOZqcqWmLw250usa5zuG8XXNPA35FuFZF2vrVBkupFvhhlRcU/BqzwWTjri4Vm7ATn1LXZ2rqc+fYZbDSaDtg1zcD4YArhtBWSpcnfchElzEBdF0XEsFaX9jLCQPmy5N+yfD9Smewj8SFgd1J5yc8VXNn2H4DwIPy1+gE6h+4FrP59LgmtkHWoLzdDT0BDUf/AAdwcP8ElCt4GpYwEBKHjo+v0IS5x7n84vBTGDbdNV3EPr3Dv7e4KHsOvI5tRt4gHUB86vTwb0ryAPt0iTgpeHMxB/a0iMJ/46b3V6T7gDrUQvDdRlf1R/9G5T4dezvGUnYw9161epgv1YxgLxABDu9gf9CI5GXg3dhA1gK7Kc6Ee6CYgPI/l2T8f37xX3jL3DM1RawZ8ZXdWm4OPFbj/bcGKznJwj9Xha/42Geo2EoWK66CXaAQ0CbK215asb373j7tWMSb3u8qMydhAvu5LGWpnRmWhvhYBwPH8JX8DyEG6M3JNNeg49BAwUjupkfAw60k3FDyNbhRAxIIrvgPgYa0DrCZC1LwtbtrW4/UBdBa3gaXIT3QRjcnfG/DyPhdnDxKgdUXoZRcAM4AfJVBQ9oj3bJgy1xzwZvj9rhWAiyH/bH9tjmm8EJsjxcDkGGw2LZH799UE7MQWD/14IOcD98DtrdONUGhoD12zZtn0uLEqmtwvPrJJnuwdWed4I2NZ/t2QwuhlVAlcFL4Dg5xgvBPPAwBB2Kx7HSHhfATKDsv+P3HtgfF7g6CeaGG2FJyNbBRNivk0HbBznfbkkCtsMLjxtSbVqRDK/WlimmzzAWcE4Mh2egPcwGXcE5rpzjzg3n2QnQEtaEleET+BLUMBiX8eX38hDZJ0FPsOxNwbr2AcvrDc/BLmBeD6sT4XtIawUC7mUbg8/vC7+Dz9dFfcj0FPyUZPbw2i3xB2ddPK5Hy5d5QVsEPZp4/sD9GmYNCdW4TxDvGncdrg0HgboE3F/t8/9gDUjvtY8TVl/BX/CyAfQNuJcF3Zh4zKPtypJwcOyPNtee9sdxLoOFIfRRtwU8DR7YtvcXuBqCbko81vMAlMFa8CvsBZaxOcwC3aAP3AkVoHaAUEYmopqX2YlfFjyX1BhYAkYbmN4qKVADNNQA6AWl8Bo4QMoNfjvYGFaC9UHjKSfJotADdgcHfz5Iy+cXAdvqQXYbaEAPFwfEAbobjoUOsDM44E6qA+FvcMJPhNXB/IZPgy2hE3wNd4CybNuyFTjwLnLz1UU7ksn65WQ4FK4AdQIsDfbVjUh77AJO1GFgm93ERoD2ag1eOJaHIMO2Ry0OHTK+qo/QtONG8BbcCy448+wJt4ALZFtwQtqOPmAfO0JaxQRsz1Pg83vATdAe+oN2dLzXhK/B8u+D5WAumAccl6NBW74K2sD+ON5qe9BWvcG2LAAng7KcTcCxKoN9wTjTXTQ+9wlky75dnR1JeBTcnMSviDsODoOTUvhsWh0JzAnaMqppWKCCbgyBwbAh/AVXQpCHlWuvPME5fBm4PidDUCWev0MAtzjlb5XyZ3vHJxGuAw/b8hSn4H8FXgDXzP3g/vAphPWON6Ncz59KSjhQk2zVOj7vGg6yf+6taZnHtV2e4F7geglKP6890jYIedLuVQTc896DveFjcK/YH4bDkvA6WE9ajlFQdp0hXjc9Ho6zfUrL/nwD5QnP4B4HM0FpCvvh3rMnqCHg3heUXY92s+yxUJ7iAPyjwLR0u2uaH2SdIp/TrpOmxPzzo7JU1PTxZk+WqW2Fh8wq4ERYH9rBHBB0OZ4f4Te4AvpCkBNeAzlpXDjrQi51JdID/7ok8Q7cMvgTFgEnZC9wkTn4s0F16k/C9eDkdWCOh1VhAVCW7aA7GV6E7IsMUTnVm9gyGAT9YA14ENS28AysBB6Cz4OXEf32/y5QF4L15is3uA9gIVgStEcZOA5fgBvld9AHToTFQVuNgLS6EFga3oEymBM+h42gLlqHTB629lWdAbtmfP+8aIvnYBlYDbTFFhB0DZ5fwbZ9BLXZv4Q8ncD+qx6gTQMXG4lWBMc8W87LtByj92F8OjL6m4wFXPMDwU/IXIPqNfgDPCjEObcEvAHO0zAH3YfmBuXeE+INr+BLLXqbdNeU8996boFNwYPCNzmuH/dL19vTsDKk5bp0TbtmwvOb4PfSUxe9SSb3unCAbYDffTctbdECgi0m4F8+naEavwe2azFbrj/7fCZYt/m07U5wdIL7n2vYevNVWfKAda8NLybh4LyOx/6G/jjOK4H7yy4pZsV/PzwDB4PnRBkE9Uk86Xos2z33brD8h2AH8IKhrcMzeDNvDE/WU4t+Jv0b8PxQzo0vwXk43dWyQC3woLkHxoET7i9I66dUwMPASR+UndaWhOxN3LxzwS96UvoWv5PsSlgdHMAXwElZDJWQS07gz1MJLv7xMFsSZz+CJuOxrLpoDzI5WezfS+DkcVEo2+8k8EAOehmPeceGCFzbPjoVTnudrNVpTJLghjYJXJBBn+LRVk7obWA7cJLb5/XBS0VQruc/I9Hn66J2ZEqPkzdtF8HCqYe1hbaePRX3WMqfbf9UUk6vdpEw3m6sYczcEJ2TykvDEHBTrknmC8/UlC+mzbgWcBMeBB5oT8Ch4IHh2HvAeFivCd/BaeBB9CQsD2GeGeccfhjcQ9xXnO816XsST4RnwPq8aHwN74F72SPQG2yDe8WekNYPBI4Hn78PbI9tfBfqcuA+TT4PM9vufumbiV0grVMIPAWW/yNsDP2gNmnTB8H95aNUZve5O8EzYjEoB+t+AAZBZ9DerWFByFeH8cBysARon3thSQhy/Oy3bfsWNoGtIFvuW3/Ac/AqrAUXQtAheHqA7XW/HQoV4L5q/uGwNri3Wta58GwS9lzoDn1gJ6hNB5HhdrCOnuBYfACNW3l8OcJD+9JUbw7H7wJT18GpGV/Vi/6LkvBvuMsm/mJcjaJRNwUXhboanBSLgvnbgJoFnPgbgAvRCacWgkpYAOYBF3OQA90RToTBENQJj5PFMi8H6wvKDof4tLsIAesMbTDNxTwBehtALpz04lyP8FbQHn6FWUHNAX+B/SiDDyFobzz2WV0A9kO5UOyD8nB2gs5rINGBuN1gI7Bepb2diEcbSMlF6/Pzp+IOwO+EV7Zt7oyv6uPT5RL/87iWvz68l8TpLAH3g+0aB+oGODPjq3rRVvZNuWAtIygd/oZIF2wuvUDkEVkJ2tT2dgH76/ivALXpWTLsUVumfNLzWEv5FNus8+ZhU/eMsA7TNmtJoAzCXJ8Nv3N4HUivZYKZeey+5MWgAmYH5TObQU8wLqyTJfG7ttUa0Crj++elFK/POffTmo/A5mAb3I+qUykJ2c87x8tAV60KoQznvXtLkOvW/iyQRGgj8wfZ/zVhQ7DPQZbTNgRw0+F2hMtAN1uLEeF+twq0SCXaDvvh/mC5pivtEtprW1Y3MpF5ta96A9YC97WVIMj+9AoB3JnAs8XxnQuqk7aznAHg3hX0Kp51E1YOkSl3Gfz2w70mLdu+NvisbVDOi9D+Tvg7GonS8Ybdix2jHgbqU3mspZqbkUdBh1DSu+ANTv9H8Dqo62AMHAXHwLfgBFK/wZuwG9wMT4GDpqGeAXU1HJbxVX3n4TH8O8NQuBY6gOV46Dgpvaz8Cg6KA+YBeDa4eGyHA+RgfA3nwB7wHpwI6nII9WWH7cOBRmYp16XBLOfCJ2A7Nobv4WCwjB/AjUFdAU+D/XocJoGXBie3fTkFDgJt9SKo6i4Nplney7ALnA9fggvf+uz3PqC9RsHKkK3LiHgFdoXzYAS0A/UXzJ3x5b40tCDNZ4fAwMR/OK4LfRyo7vAjnAB7wheJi1PjpcFxGgylkK0ViRgNl8DuYLu1t/Yqgc5QCafDSVl0JRxk+21nQRdqHmsptCO6tVhgOtq0gqbNXkvzYnLDWOANqundAFV5aShrgHqmSxX5rKWWBWrhRZTzMywP5eBtcXMoBuXhPAFmhl7wFQR5oKwGz4IHjRv723AmqOvBg1MNhG1gcbgDbgcX8JqwCbiQd4EuYH1eGDysPRhbwX7wE3go9IQdwcP5EHgC1A3gJSQoHX6ByIkhIeWOwT8QstM8nDzoFoAHYX2wncWwLrwLal+wX53BZ8KN2XK1V39Qm4EHrroF7J86EDyEg7wU9INu8BWsAGPBPm4La8Fk2BS0dba0U1+wrlHg87+A2gO0nzoaLF+dDvbVci1f23aAE+FRaAN7gbLfjomXvAXBMXsW1BnwfsZX9ZIOb0+U/cql14hcBraGheFb2AjeBPUnDNSTQ87dIOeo/f8gREQ3WiDLAkMIZ6/1rCwx2EAWuJ96fHNQ33qAChqinvruR/2Wn8/to4aWXEfa/tWkezh7aET92wJekrzMRDURCxRoLTURaxSmG9GmhbFjLCVaIJ+1VNIA5vqEOr6ppp7niP+7mrTmHP18tEtzHv7Y92iBaIFogRnQAvncPmbA7sUmRws0mAXiWiq8qaNNC2/TrBIXIeyPfKOauAXyWUsN8UlDEzd37F60QLRAtEC1FiglZWCSuh7uy4l/ejmDqdjvUNWmO8nwFvidpB9gCSiUxlCQF5Js+Z2BJbMj8wivTd41kvzn4tr2QsrvSfm9s0KrnAIL+sXrQjcwXV68NKStEf3RAtEC0QKFtYBffO6dFOmXdetyYBe2BVNXml+E9ovUR4BfPh4FjV0H0UC/BB1VjxZobJeGx+nrbNX0dyfin4ZZs9L9DZB74Pgk3t+vDb95kURlnKG8Lp1E3IbbM/EHZxAey89FO+L9Rr2TMmh+POeAvwkwAh4CfyMgqDWepyBXed6yB1WTZn7rawzqRSPCx5NX4V+uMTQqtiFaYCos4MF9F/gbRP7Wj/uG2h38LR5/E8s1vCUEzYHHNe6avAYWBdUDDoAr4BZoAV4O3GPMa7xfZDb/DuC7X/N7oG0Myr3XC8QjcB+EeLyZv3mzFq7f2DetDyjrOQp8ZhisD7XJeg4F+30hpPdX17Z7pW2+HjqCuhS0j313z9wa2kIXcA+0PPe2ayH9RfbtCGtD2705BM2H5xJ4Evztq5rkpwWW4XhoZ3UIGB/UDc8xIZC4G+EuC3vDmkncvLiOhf3TbuG8a4P/5CT+RtzqPkWZmzRtZl8dq7R85gaw7FNgFpgZnCeWH7QXntD2/vi1jf3TprnUncibwHJPgzBe/fA7l66DR8F+BnnWHA+20/m4DKj2YLy/2eZ8mRPqV/n8nKNALZlAOe2qKcsF/zf0zEp3MY6HI5P4wbgaKS0n9iRwkWi4yeCESms5AmXg4q8EB6kMeoNKT/gOhL+Gy8GF5ORyMY2DxUCtDD9DWQ5aEFdbfWSZ7vqNFmg7tSeETSUTEV/qboHpsJbq3rgZNGceNl2eLvoR+xbggeOh6yVYDQbXsuvcNTsC1gX1LJwHrmkPjS/Ag8ED/k8YBB6Uq4Jl9AIPkzvgRnDDvxA8JNzI14OXQZ0A+lcA6xsF7j3qF/CNkHvEfjAWZoKd4THoDGuB8bW9s7Ye++FhtBtMgHAA+ibNw3xx2APKwTavDu6BG0A70Hb2qwz+guOgK9jPW0HtCu+A/dEen8BGoF6Fs2BpuAYqYBHI1vdE2FbHaCewXvfWPeFhCPJSc3gIJK7lvQCnwEJwLoyGvuC4fgFbgxoKtkM77gjafi7I1nNE/B/Y7svAdvvMfPAd2EbtciXcAUqbbpfxVY3Zj/g9L7aEj2EVWBHehQGgyqEH2Afz7w5LwRB4FJSXHMd7U9A+b4L51E0JtsU+fgMLgPn+BMd4D6hWeaylasvIJORZUCceOgxOAid3K1CrwWLgwJvmwgoyjxP5RHBSO6GdpLn0FZGvwLapRPMa/x6smcR/gLt54g/OZnjeTwLr4H4ZEnK4fYhzQqRVQuBXWDaJfATXyZStq4nYJoncH9fNojblqi/7mU2I0HZuIHMkibrGu8BNc/K3gKCOeI6EY6FriMTVNk4mn9HmLcEJbfhocLIq7eSEOwisy8nqRFSzwV7gM/2hGJT1WLYT1zSfCXLxHwDGOx7hGbxNX3mupaZvkAL0MA+bummelqpyQfzuNbPCYHCdBB2I50ZwrbtJu45KE97Ede5uDJ9AkHN7ySTgGjkVnkzC++Nel/jXww2XhpH4V0ridfaGh5PwL7jdE7/OOFgI3EM/hH7gGrT9telzMqT33KGE94UuMB4Wg9KEl3Bdu6oSrEN5eHsglUF671yX8KugtI19KE04Bte6lgafLwY1M/wBHpDZ+p6IsMeadh/sBm3B/Xd+mAnMF/YivFNk/m2S0Lm4F05Jqdqv3Xvaw0ToAqUJT+DuCml1IuA4tEgiPaucD53BfcxnShMce8ucC7aHh0D1hcczvqKi4biHQ2mCZYQ5Uo6/BxwBN0JQazy/gXWeDJdC0OZ4ngXn3mRwvpQm3I9r+d3AueMeX6PyWEu1F1ZjTf8kOqGeg7NhFBwEXUEj7AZ94CZwsmhEw2/AndAC7oXLobrOOUFcNNdAWJx4M5cNy9gPLG92cDK8BmmtSCDEuVCDP50n+HOlW6dtex8WgzLYFrK1RyrCcr6A0lSc3h9hfCou3bZU9BSvk39NuAKWBxepZS8I2tSL1PWwC6wGe4OT5RH4P3DSPQg7wTNwKlTA8+AkvwpcjHeAk9PySyFbg4hw0rt4X4C3wUlr3AawJ2wEjv3d8DFcDYfArfAYPAyfg4tXm54DUdEC9W0B9493UpV8h9814PxX5ZnXqpdvcVxb7aEEroMg536QZQQV4zkZesNI+BmMq0nW8VUqQzn++VLhMSn/RPy25Vqwza6f28DDwf013S6C/5L7pn0KKk88C+NqA/fUoL/weADVpNGpRNsV+ml5tmXrVLp7gPVrq8ok3jq+T/y5nK9Tkd/gd29yD7sP3HPL4Q2oqQySM0rb8G9iWoDjOgkuh7TsS1q228tOsIfptkdZxpLgvhvkfurl5h64EGz3jjAYlM9sBxsbSORcSct5mp4TtlnbWZYqz7xWvYR5ajtto3WmpZ3VT2B/C6aWBSrJCb0XDEuVt3XKPxS/E115SHroeXCuBnbaAXkU0gYjOEU+44H9FuyQxHbG9SbnIbUhOLH6gMZMLxKCmTpdYMqyHODqZLoHZ1rGWbfGXxnehV9AeWAaF3Q1Huv3GSfRJpDWlgSc9EHaorpLTDvS9oMO8CNYtv3YFR6GVrAV2PeHwHqPg2PgEvDCoWzrsfCMAeQECxO+C/7bwTzOh51gMXgCHJc74FcI2gKP9e2cRNyJ62I6KwlrpwMTv4tgFbD8peFEsN1PwhwQFS3QEBZw4+2UqmjhxO+GqhapcjKv5isH19IEWA9cB2pTcK0uD5UQ5HpzLS4K7hG7w/ZQkyy/I/yQZLJe11FNWo5E99j/gX3wgLKey6A6uae6f4S9tT3+z0CbeGlw//JwUhvBOxlf/i+WZ7vuTh5dCnducN+yTg9sD7eZYF6oTtrQS5fSJi9kfFWXt9Pw248hSVwupzgVmR6jEG07S8CxHJdEOsafJP7gjMSzINhe54Htdz9TlqGdwt7u2A8Ax28i3AXOgdVhB1A+cx1cbwAtDul5Z5x5eupJNDOu5+PXSTidX9uUw/dgfzxXRoNaC7RTG8hlA6KnXlZWCH1OIaXwCDghj4N02XYg6Hc8DsAS8CloZDUKwmTJRKRePIBdrB/Dkkn8ObgnQTcwTa0Ar2d8/7xYlwfXS0lUKOufHP/25UpPH+zzkH1M6pHx+MvBhXci2D8PRPvnIu+YxRuE07K+V9MRKb/POilceEE+3zkJaGsPcKX7LSwGS8PBMCJBWwW74c2Mka56GS6AN+FtcKGnx47gv2Td5g2yXsffRaBGVjmZVxel9ndz2geuACf2GWBcVLRAQ1jAzXoP2B7clN24B8NfoA6F9WBjOAiuAi+/7k+XQ1fYF64BD75s/UGEa74UyuBwmAWUa6A7rG0gpSvxXwqrgvW6ZxpXk9xPbgDfpCwKc8EnoOxTqZ4sWcf54B7mQWZd6kNwL7kauoH2GQJTq8t48Eywn6vBPeCeow3dLy4Gba/bBqrTWSRoL8ekC9wLajjMD6vDfZBL2np9cP+rTl+R8Dhorx6wE9wC7lNpuY89A/bLdrtHOsbqNnAM3GOdGxeC+9tEUJZ9LAyFP0FZziDYAHz2TrD+tG4lsC4415YF59vzUA5qIGwFveE0uBx+grthMPjM1nAHtIZsbUnEftmR+YZb5vtANfmPIn5T2Bveh21hZwiqDJ6U+wv+uVJhB23WVDjtXYnAXTACFgQn5UJwIwyDx0AZ933G98/LOnidTG/DwrAAOIlzaT4iO8BrWYkrEr4giSvHXQKKwX49m9Af92Pw0rAWOOl+hJoU6su+6IRnfsPTDkqgIon0UNd2ao4qJ/Nqe0wbDbbhHLgZlBuY5QSFjW92IlwYh8Nh8DN8BTXJNnXKymC9Y5O40M50Ftv/FAwGNwQXhRN9KYiKFqhvC7xFBe5PB8A2cD+4iQd5aOwErWA7eBXUhnAEeCh4OPeGn+BreASCzsJzHLiJvwP94GBwT7sXeoKHnXP+QVBngPuDz7l2rP9JULbnj4yv6iWEPUTawIkwAY6G8EwX/HNBOaR1CYE/wWc+hYPgC1CbgWv//8C4MvgG1GCYqAfdBr+C6/gBCHKvDeGr8JvnQPC5U8F2qy3gGLDP7uPuIen+Eczodl6fgNNB22jv30G512rLWeAvyCX3vP1hKXBPtd9Bb+CZlAQG4B4G54L7nWfEl5AtzzFtbLut+2yw7T/AquDc2AAse1MIegWPY3VtiMD1QmCf94IKuBiuA6VNxoC2t8+Wa3nPw/kQZD7bOj+YZxioneBQOAssYyP4ENrD3dBwyuPLEZfSqmCAdvifAw8jZfz+GV/VSwjPRLActqqKznTaieHz2fqZiO5J5Pu4L8BqSfg73BUTf39cwyslYQf2S9gjCffF/QxKs7AtamMwPS3TnKSLJ5Gtcd1AzoO2MBusD6/C9aCcaI9AaRYdCKeVq750ejGBd8AFp7qCE9b+uUlor21AHQivZXxVG9ab+OcB238LOInVu9Ar46u6gLmQFknCLhLL7J2Etfta0AregDVhSRgNwcZ74R8BLeFIuASCQtjLjRvjckmCY/5e4u+B60Jo0spjLTVpOxSycwWy6WDatG8h2zWdyvINlOusqWpmOvY2hD2kqfazun6dTIKXgnpRPmvJjb4Q8oZ3A3jA/QhXwiHgTduDJ9wW8U4JT8C/GXg7Px/uAydFBaTlbckyPkgivTB449VdAMaBB6G6B7yFecPrCJ+Dxh4CqhPY56cNpLQyftu9GDyaitfbGT4Fy1J/w9rwP/AC42Q2/Ta4GpSHsIdrdj3fELc6BNme7PpCmq4HeD/wIN4fbKPuq9AlCWtD7f8hhAvYRfjbgTdex+AZOBjUt/BXxld1wToO/1MwBp6Fq6A08dsfbTkAwnOf4N8FrgBtbx21pAAAQABJREFUrQ02gEnwK1hOUAjr7go3ge3ykrETqD6wLDxhICpaIFpgqizgBd111hS1BZ1yL3J/f6spdrDJ9Cmf20eT6fSM0xEvDaNmnOY275bGtVT48Y82LbxNG3GJvkGMqicL5LOWSuqpDbHYaIFogWiBaIFogUJZ4PtCFRTLmTYLxEvDtNlvej49ksr9sUFUtEC0QLRAtEC0QINYIF4aGsTM9VLJeEp9qV5KjoVGC0QLRAtEC0QL5LBAvDTkMEqMihaIFogWiBaIFogW+K8F4qXhvzaJMdEC0QLRAtEC0QLRAjksEC8NOYwSo6IFogWiBaIFogWiBf5rgXhp+K9NYky0QLRAtEC0QLRAtEAOC8RLQw6jxKhogWiBaIFogWiBaIH/WiBeGv5rkxgTLRAtEC0QLRAtEC2QwwLx0pDDKDEqWiBaIFogWiBaIFrgvxaIl4b/2iTGRAtEC0QLRAtEC0QL5LBAvDTkMEqMihaIFogWiBaIFogW+K8F4qXhvzaJMdEC0QLRAtEC0QLRAjksEC8NOYwSo6IFogWiBaIFogWiBf5rgXhp+K9NYky0QLRAtEC0QLRAtEAOC8RLQw6jxKhogWiBaIFogWiBaIH/WiBeGv5rkxgTLRAtEC0QLRAtEC2QwwLx0pDDKDEqWiBaIFqgwBaYhfK2LHCZsbhogQa3QLw0NLjJY4XRAtEC09kCvan/gTza0COPvNVl3YmEd6pLbID4banjinqqpxD2qaemxWILbYGmdGkoxjirF9pAsbxogWiBJmeBZ+nRa3n0qm8eeXNlbUFkR/gsV2IDxd1KPd/VU13Tap96alYstj4s0LI+CqXMMtgK9oO6aF4y/VSXjDXk2YS0X3Ok70Xc/OAF6R54F6ZFc/PwL1AxLYU00LMzUY8fi+aySwM1IVYTLdAoLNCVVmwPrlv3Gg/yoOXwbA6mtYJL4XtQO0KZnkTf4l6d+N1TjgXL0u9FJNcnGO6Fd0FavQhsA+PgU2gPr8ATUAYbw3kwEGaDZ+BxcC/bByrBeofBmzAXHAovQm9Qf8P/wVgDyP3gKJgZfPZ0ME9tmpUMR4B12s8xcCGoHaBMTyIvJleFAK4XitVgMmi7i8FylO2dCI/CbqD9zwLbuxTsAtZnnougtkvPkuTZEx6BDeAP+A0ugEmwFnhOXA6fwRqwBVwJH0HUtFqgTZs2YXCnpqiT8ngon7zVFXtujoRexPXLET8tUU5uF/iMoO400kUbNZ0tMI1raTq3vnFWn4dNPXTPAQ8g1QXeyfj+++LBemJWdF33p1N4LtSRLiJ7b5qPxLNTGTbEPyQV1jsQbgI/oVBzVjn/eT0tFeNlYSh48VHzgBePoGF4FkoCi+PuHhJqcb0Y9EjlOTDl11udfcpI801b0Mp4DgmBxL0Z93hoCbOA9m8LHvRebFQb8LJRbKAWHUD6qak87oGHpcJpu1tfdW1PPdL0vXmspcxAFcoiDvS+oPt0VqHzE/ZW7cT/Ga6BcMPdCX8ZBHnD9+YX5E2wK3hTvRty3QhXJ/55SGszAtbpTdOJ8yXcCMrJOBA6wZ/gO4tfQDlBfW4RMG0w/AhqHbDcpcFyJ4Ebgn3ZEpyE1jEXOHmfgeGgtgPLPBt2hEXhEhgLbjQDYTHIbg9RNWoZUrWR9vH2PBxs71KwD7SDsOCvwz8KlBuLC3phMP9lEMakDH8fcKEuD2vADfA5qI1gNaiEwRDi8Varw0l5FUrBTXsMXA6+G3BeHASW4yYyOxwM5aA9o6IFpsUCa/Pw7VCRFPIx7vDEr9MZ9gTXnmvA+VgXtSCT83Re+B1WBeNCPXiL1ofH9KTUG/9tqfDD+N1bsnUyESOSyLGJa9tcK+411rk4pDWEwMQkwr12PBSDa/Ut+BbUSLDdddGDZHL9bgra6C6oi9Yj07GpjK/g75sK6y2H0/Qg91O1LswBxxtI1AG3I3wZImpwr02lvYt/QCo8HH8feAYGwmCIKpQF8rh9uFA8AL1Bqy3g5Yyv6mUhHA8vpf+IjO+fl5P+8f7L52RfOonxYPWANi5bZxORK95J3jM7M+EzoEcSb5s9vFonYQ+sRRO/C9O8ae1GoH06IuU/LuXXm92vU4k7D1YAF/+SoE6HZTO+qoWcbk8SXa1zMSktk9QDcD3Mg+xj9iINaRfiWSoJuPFcFhISd/skzrG0fNusdgXj1Myg7TsaqEWtSP8UVknyLYJ7ReLXOQzmT4X3xF+XclOPNF5vHmup8XaikbUsD5uuSdPXyWr+kCRcjHsBzJRKz1632eGQ9WA8XoCDPCCd52m5Z2VrZSJ8YxI0N57BIZC4A3E7ZMUZdD/yTUlQum1eRo4JCYnrOg9K57W/2XlDvmzXPTtoNjwXQdgvjU+XazjoFDzuEWn5iU9auZ5diQxbpTPl4XcP3DCV3/HIrvNM4mz/yal8zdqbx1rK+VHa1BivDw/dAz8mD9+N+0Hi1xkL/cEJ4mQIlwu8NaqSVBfICeCC7ADpxU2wyHfaH4F56yLLmxe8xJSBlxLbui6ocbA82FYn4IJQKHm5csK+DtrkE7Ad2mNOKAP78z6sB3XRaDIdDh7kd8ELUJs6k8E6tJvyHf4z0NNASrfhdyx9B2CblZ86WGcZeAF4CnaAuuhhMr2cZByF+zT4KZC6BnbP+KrGuD3+EUk4OtEC02KBZ3l4cwiHn/PVi7LrzkvDJPDduW8SDoVSmB2C/sTjfFQLQ7iYe/D8aiRaHVyD6f1iRcJh3eCdolfweWH3DcimMAh+giAvEfOAdZZCWwiyveOhBLYG22L+IPezI8FLknvNg6DcX2Q2A8i+u/d4qNam08jQLclUgeteOzkJ69ie8EZLG2sLdT3YhjZguw8G13xQKR7bpCvmUa+Ce0vYGxyXPcD2Bs2P5zfwXMjWhkTsDhvBBXA1pOV8uBJuSEdGf90s0LJu2WrN5aB6kKTlRAo6Hc+1CU7ao0JCLe4A0r2png1/w1mQre2JGJQdWUPYCe+CS8sD9OMkwrY5aU8BF5QLplCyDz9nFWZ7vEyk5SUmtCcdn8s/iEjHcUnYHYbDc1CTtOm4rAx/EHYc0/o2HUj82bb7i3gvF3XRrFmZDE9I4tx8/wQ3s80gLmiMEFUQC3jAHQ27ggfP43AYeLjdA4PhGHB9Gh4JXeA1UJeAz3rAfQ93gboIPPi9YLwHu4Dv9m8CtRVYbi6dROTS0BY8TI+HIMvwErF4wqe4L4HyEDwA3JvsxxmwEjwM6jr4DHzW/XIsqO7wNiwBb8LK4D6zAIyCmnQuicvCpuDB/n8wCYK0j3aYE36AO0GNgPPgQHDPuQ+sW80PfcA26aoHIeyPR+LfFvrDeHgAfoGgMXjOhydCRModhN/y5wLH2T0qLZ/ZGL5IR0Z/ASyQx0cWHjZOpJZJtd1w30n8JbinJn6dtcBJltaJBFqnIhZO/C4sF4dysT8F6XyLEj4IqpOTvGeORC8hPhvUHk+nJHB6iMTtDPemwnq3hh6puNBWo2xv0DZ40mHjs8Mhr4u7uvaEPLncPkTumEqwD3tnhV3MQb4LcNHLFTAHqNnBcLC1cduDG0+29iGidypyVvzLp8J6j4BbsuIs2w1iB7A+2+5mlFZbAifAaenIpuDPYy01he42SB9mAJt6KaiL3C/3rUvGWvK4LtesJU9TTz6ADnpZqEk7k+ibrKjEAvmsJQ+PamVB48ePrzFP6mEP2F3A27q3zd/Bm+el0Be6g2W9m/hfxH0clAevh5E3whZwH7wF6XjL/QY6wJmgTgJvm9aVLdviu4VJ4LtZy7NcNRMcCLOAh9kYuBL+hBVgc/CZH8FLjzfdIaC8tBwFFWBb3wBvwaoHbJXxVV1wNsV/Blj+0TAPjAUV+qi/pvaYXp081G3rbGA/K8ELkf6g/fB44bIfvku6GsxnW1xg5rV+x8l0tTu4qLT5RHgdHoSggXgcB+Xz18F3BhL5LmE52DZE4GrnI+ApWBs+gaHgu8C0vEhcAx+nI2d0f55raUbvboO0fwa3qevRPcF14bq7ClyXUyvXuOtOvQO3ZXzN68VLmm+SxoH7zDOQVhmBPqCt3H8uhygskM9a8hCvVvkUVG0h9ZuwAcU/Ur9VxNILZIFwafASVZ28hA0CP21oUpoB1tIMZ+9o04IMmetyuxpK8vLhm4eoJmyBfNZSyxncDvHCMOMM4OE01R9j+G7IT0PSakvgEPDTD3/cUQrlEBUtEC1Qvxbwk8TwKWr91hRLbxIWmNE/aWgSg1BDJ/yyzgrVpD9K/MvVpMXoRmaBfG7yjazpjbY50aaNdmhiw2YwCxRsLVnQDNb32NxogUZpgbiWCj8s0aaFt2kssXlaIJ+15JdxoqIFogWiBaIFogWiBaIFarVAvDTUaqKYIVogWiBaIFogWiBaQAvES0OcB9EC0QLRAtEC0QLRAnWyQLw01MlMMVO0QLRAtEC0QLRAtMCM/iuXcQSjBaIFmq8FRufzBa7ma6bY82iBWi0wutYcdckQF2RdrBTzRAvUboG4lmq3UcxRUAscVNDSZqzC/Au5/rXNqHqwQG1/p+Fn6py7HuqNRUYLNDcLjOZPsvvHq6KiBerbAotTgX9K2j+U1hzVn07vCf7F4KhogWiBaIFogWiBaIFqLOD31L6A5vw3dq6n//7PnahogWiBerTAsZTtP/CKihaIFphxLXAJTffC0FwvDV6a/IeJi0JUPVgg/vbEtBnV/6jWVOR/5FyxqXQm9iNaoJlaoAP99sfKzVWr0HH/4+7I5mqA+u53vDRMvYX9J0sXgG5T0At0YrWm0JHYh2iBZmyBP+j7mTC+mdrANz/3N9O+x243cgvsSvv8CHBgI29nXZvXj4wP1DVzzBctEC3Q6CzQmhb9AvPC+Y2udQ3ToPepZqWGqSrWEi2QnwWeJLuXhjfye6zR5p6flvm7ujX+Rk2jbX1sWLRAtID/FXd4YoYFmqE5OtHnbyHuYfU4+PHHE1NnXH8ksRx8Bf56U1P4EYVfHvLSsDRERQtEC8x4FvDTwqFJs7+f8Zo/zS32RxMPQnP9Eug0G7AuBcRLQ12s9N88axPlAr0LbgfDTUHxew1NYRRjH5qjBVrQaQ/NYc2x80mf7X/8PkMzngCNuettksaF7wGEcGNuc13atjuZhtQlY8wTLRAt0Kgs0IfWvN6oWtSwjZmD6n6FprIXN6z1Ym0NZoGm9j2ApbDc5w1mvVhRtEC0QKEs4G9yHV+owmbAcgbQ5vhF7hlw4Jpjkz+l08s0kY77BSJ/x7s5fomqiQxh7EYztUA5/W7O30e6kf77p6Oj6tkC8TsN027gpvQ9AL9A9CLEv9cw7fMilhAt0FAWWJ6K/oYPG6rCRlaP3+fYEPwSZFQ9WyBeGqbdwE3p0qA1mlp/pn2EYwnRAo3bAunfmmjcLa2f1vkmpxy+qZ/iY6nRAoW1QFP7HsDqmOeVwpoolhYtEC1QjxbwDxqtXI/lN/aiz6GBJzX2Rsb2RQsECzS17wHMTMd+h/gt5DDC0Y0WaLwWWIKmjQL3oeaqj+l4z+ba+Ybud/zxxLRbvKl9D+AvTPIuxD/FOu1zI5YQLVDfFvBHE/eC+1BzlJemWeGt5tj56dHneGkojNWb2vcAmlp/CjPKsZRogcZngeb+fQb/oJO/atlcL02Nb0bGFtXJAk3tewB96fVDdep5zBQtEC0wvSywMBX/BC2nVwMaQb1P04aNGkE7YhOiBfKyQFP7HoD/JW8MxE+i8poGMXO0QINaYD9qG9ygNTauytrRnLHg/hvVQBaIh0JhDN3Uvgfguxdpzn8spjAzI5YSLVB/FmjuP5rwEwY/aXD/jWogC8RLQ+EM3dS+B9DU+lO4kY4lZSzQt9tt/kGdf6nfMrf1CBEbLn5T28273rpqCM+Ibt9lby3dotvNnRph2+eiTSvAY42wbQ3VpE2oKP6DqoaydlJPvDQUzuBN7S8pxktD4eZGkytp8+637VRcXNQ13bG+XW/djR9o3RziHv58h9+Ki4uP69/9pvYhrtBu3663Lcbl5aJClxvKq5xcvMvkohZ7hXAjcv0C4BPwZyNqU0M2pRWVrQd1+iuQfZe5Y1nmyZ42cNPlb5mnb/dbz2Du3EXceVxsFzF+s+VuXahvt9uP199YxDrbt1+3O/rk057Nut66Qr9utx2azzP55I2XhnysVXNeD9lVoKnYNF4aah7vZpu6Va87ZimuLDq25awll2gEP1Fw8y0qLj4n2yiVJcUXVRS1PCM7vlBhyudHaMXLFKq8Gaic5v6jiTUYq8/gh9rGbFDRoJLKFhWXlEyayKcSlcUtJpQ8hjNLZVHl+cydP7jYPrPh4g+1vu+tbb8tKqpYun/329zHG4VYZ2UVxRWd82lMSUlJh4qi4nXzeSafvM35W7f52KkuedPfA/AvtM3o+oQOtIUF4bsZvTOx/YWzwIQ/KrbnLwm9dOdLAzLvclvP0jLzv0oqi4oHFBdV+t8Wp2imJYqfmPhx5XVbLXPHAnd+MOD7KQl4+nW/7amiiuKXKoordy2qrNy3ZXHFO5OKWl5KGctR/g/8Et3JQ9/f5h6f6df1tv6VxUUn4l2Aet4qrqzcv1Xrkp8nTqi8hs2/bd/ut90/7N1tfPc9RbyTfIRPQx6rrCzam2dnp9zLhr237alm4B1l36LiypOow/n9TosWk/e/++3tP6OdM00sqTiP+vvxO3zfFhcVf035Hk6094b5iipbX0rYv2HyG3lOGfreNneWlT3dcs6ff/DTjnX4E0s8VvTAsu99fASHVQX++tCsFFoGA6G5yrGu048m3u621GbFlRUj7/lox+/6dy9uX1HZsmjYe1vzTpwjmf+1w4V359azjvPHaq8yelcxaCfg3xj+JebGXJNKKjbkctxyUlHxs/e/O2CEGbyUvN21yzrFJcULMo/Ht6woeZi5Pq5fl3vmLmn117zM6VYlxZU9KyuL3x/23oA3Nu92+0YlxUXzFk1o9cDQj/uP9pOPln+XzFvZomR25vVSTKH37nl3mzf/VXkS4FOHXjR7icqKyu+Gvb/NlB9NbbX8HXNMmDB5Y+bfBLK2yvVsoeKayrviQtljWstpSu/OMwsKg2QOhGk1THy+6VigpLJ4Iw5MPxrPaNh72zwMhxUVV/iXRP+lO+8cMJl3d89PalG5/r8SCHCYL1xZUrlYZXFJWUXryucqilrcx+b6FGUtUFRRtBMT8Ere9fXkctG1srjyqoqSyl1MKymqfJzN8cGiTkXjuAzszsH+YvaFIVNXcdFCHPArt5q9pEdJSVEfDolj/H4CFxD+9HvltRUVlXtYHhv1g5Mnt3jIw59DwcNkyZazlXQualE5gDasGNpdWTnTzYQ/4pkOk4tLNuPycv5mXW/pPtfP39O3ym5j55l/6fHFE3qSv+ebXZfUrS9tQMEvg7850FxV50sDc3UX5shQDXXPuzt8zfgxNpkLQ1H/pW7uQPQ8xX+3/ML0Zd//5FnSVvCSazho0+Xvb8Nl8g3m23JM3A4tKiteCZ9IvN2ty/18vnxAZUVRKe/wdyXfMz43eaa/V5tcVPJgcVHF/7iMMOcqhvfrdvu9JG3EvNywsuWEl5g3xS0nlKxWWVz8WFFF5WmUMQ+Xlnv6dr19h1B3cP2RSmVRxcUVRRUL/z97ZwEnR5H24e6elbhBcEkgEDeCa4IeGoE47u4ceh/BOeSwuwPCAcEiG4hhQRNcA2Q3CoEkQJCQEJeV6f6e/+z0pjOs7+zubG+9v98zXV1d+nZP1dvV1dVc/9f06zrmOR2T0ZGf537NiMmhtmcf7HkaQak+MUZDcnUbJqNBmglbfZJ7tutpajR47XkK90O5q29b37ue16G48DSoI6dkD5pv56dtQYe87cTswfcp3MTZQ2ayGe+59omu5XGnaE+cMnNobNW/iTmDH+BYi+h8V3eHpYttjdSIyMSZQ74l4MICmzs621aHM2XKrKFfKvLEWUMfZtOoxYo/9qDB7kvH8IjiTPpm6CIMoaKGmTCHOXQWzN3oHXGjO9N5fB2xnX6WY//K3Wf3Fst//2djK22frTJWHemnrfSrQer7owm91aVR8uzy6JZOdH8M1L/cueuxmhtxsui4/6U7fqXFqAGXgDUzP+JtdrNk565ph3+Gk59/P6NLt9qW0zcfIxnjogkGSU56e6ffpFmDb/HyGgwi3B4yMtjSt1uZ6R2cE7hmr8NweINHDb9Mzhl8cY/secM4uMOAjs8XGie2F90qY+WxpHE/nfIQbJrYiJjSkBzfOWsn7ItLrEjm4ZOzh95JmhiO9oH9umb1Sst3zsME+gDD+WwMoksJPjIWqZp+jNGQXMWGrZMNW32Se7brb2qN6OA1DFpeyaNTjTWiiRFofGN3ywzXNufYChpCki4UHH9ioDTGYGhOR/6n768wHFsR5dgmv+JdthdZHzgS5eE2gxmxvALpxUL86RaQl2U1J/FVfhzyXiZ3RtRryabAtRzeWHAOKcSegVHxeWwo2XEOp1No4HnOE0vzmv+kyWh+GkneppOe3lqZnOR061Jy5R5lOLfX4+l0zlvkbizQ4+Mi0eRcHqu9z3X5MR3t/xUdwME1txSDUI+tigQjMIcjb7lp6Yv7dh37iWtFD25QkLZYjyGiXnR0/lz3fj0OczI2fKFI9lo3oi1pLSgcbcPPtlYzShd71BU3TjZGI2kyfhTw85EzzsuX889WW8uYbSODRPsSJ+LJQHasaO5EHqdMy5/naqSvGSMP3VzXYiTOiuWrsBixH8e21fRjjIbkKjY4DyC5KddOarp4GVaLre1eOyUwuaacBuiwf+OxwpblLRgd6Za2a/1WWvhoWsEPHN9OQ61+OBrZPWzb+44GUQ1t0XD/sV1faEnnvrMXsWJDyn748m49290sPT0PJu4utu0uoIOZ63ixBjqWHHnrrtZqujLvJzYF0Uj0Rd1Rxu4qbe83GT0ndB13sBX12vCM+SI6oF0JN9l2nFMUrxrkUNKcC6XqsxryTaUky200jJxxbgEFz2vQyG3gV0CPp5jX8DHn+mnO1xVBQ1VhuO6Y6GsHjU35eulu5Nx019kGA/chRhf650fcB0/onNXOsZ0PeDQwOxpxr+LxVC8/n8KtrfyLhOtJIxl/FQxO37P1H380w53/R+vWG30/x43KvZRHZmf4uK6zj5fX8EXbsdbxfygyMDzPzfTjVcfWGA3J1SrtqSUr74DkJltrqeWS80zQxC8jRgMxDTiW/Ynjej3Lq47YxEbu6EoL//KMYcu42xrJzPapPPc9h7upxwm/a1pG5PmMqDOWP9YOPMN9klfQzk63Iq8T9mnNdo9ENQLhdu/fZcylpaUfPLbezh/P/lbk8bReE83Pi77O/vOTZw39iXkOd9Gw38Az5av6dhl7O3ec/RR31KIzaLTtOx3XeZE7zfN0jI7jZuyIny0vupIO6JG+XcdcQZrnEvwwyveG4rH/wgmdx/WRO0nSn3QmJimtupiMjMrOML18hY+NSi307AZtFP7Ibs825vy+xlD/k5bjTdY6HOL0Nk8Xddqcu52xHDYzSPt3G30ocxWyV0U25k7MHjIWw+Ilro2mjhN7s+HP7jlzn3x55rDZjayMU5VPNGNt4QiCdsojttXnxB4v7KagzKu5mM1r06f3KTI4ovmNP8WvmV1gd4k9NmOHvL+wM3KZR+FN4f8xXMav5uUwmnEmh6tNjNGQfNWGbUg/bPVJ/hmvdyna43lOfGxitd10Tw3tjUH/2BsHzFVI62B9GPSXmzSui0SdRb4/d+qX8ybFrTwe2I5O+JvcDQV7jZ8xaJWGgPM2FOzDXIQZtmtvz/YO3dUr3oTZgz+ikb+CVy/z/XT8LXdf/2Dwd0Hi/pvZp65z8xruSx6f0/TuQOf/T+44z1M4zUVIs9w+HGvIHdwSy40cTpgsHWPG/e2MfFzMXeg23Hgud71oT02smzJrWDZvgRzlxB7BeFszaXM4z61fUxwMje8jduzRRuFu1X7VXp8A9dlo0HX3NpT78RjXwVTbdQ8kjtXQyzxEW87f6VbUnuazunGj2AiB5jnIaGidsWIzI3di9rB3ifVWIy9jIYbgQq7RoZGIe1Puhmbyn8tkyAX46+2LduxPT3fTd4643gY689/Yj4nrWcu4flf7+2x/jEQLCrRPuJ+ZjDuOt3p+wn24m+ZdGAvnWX9w0tdOmd93DX+YEzF47lf+dtT+kOv275NzBuVMyR46hRQm8SbRty3+/H0B8VYwoPF7LL75qRMa0CgDjVFoRI2U7sSMGA0UaYCG62291VDkUYKDRWauI6zunOqlUPeHE2fiV0ERalu+qUL8MEQdTyVOq0hFTuiW1Z7z8EV54miUi1Gke0oPS9edROE/0pfyvZXEJKs1KTPSkHz1fkmSYZoHIIt7XzDXSvKvlTqbInfhF3K3dVZpFdAiUDSvPZjp/Whp4cJ8jLpfkbg+RRXqW98fTWSgO0Z+yrcKpK9nvZ3DhMSveWXxMN+vuO3AgVkRwp2Ske7cW9zxTX6bJutu8jMuo4GqaUBDscl8jlm10lQ9tiZedat6MiYFowGjgSpoQEPP9fl/eCT1V9taYdFjB+7oB5YW8fhuWW2TPP+ktOyKjul1St68UN2M1GMN/JO63xSi+v+PulwQovqYqhgN1DUNyFiQ0VCf5REqf219VkAq1N0MOVfPWQjb5MGw1ad6zrpJ1Wig+jSgRxOTqi/5OpFyuV+1rBO1MYU0GghoQK8FrYCwGGW7U5eFgfoZp9FAlTTAa4vPa4JalRJJsch6Jl6Nw8yaAKmJkPVV6uxIy8CeWa37ds3qGpYTF5ZOLdXOxzIKpFdtypxdnmoFL6E8WoK3EWxfwnHjbTRQIQ0w/Xwfy402rVCkFA/MqoCHeU7so1rJLukuJLgVfJLshOtQenV2lKGgwO0d+/5EHVJ2aUVNK+2gOVYlDfhD+uVaH71KOdVM5I/JRnc6WTWTncmlpjTQt9uYfWzX+RtrEOTywvhkviypia+WXhXMc6JDbc9pxgqQf3i5DZ/T++L9O4/tbkWsKC+799JHeqyI/Xp6gbWgIOKeRfyIlZ/xhNby79c5q4fS8Rx3H61twHYq6+Z/Jr+gaJ3+tNz1+kDV1vrCn/9ly2AYTWRr0CDtYMt21rC+Q2+Wb16c3iHynL9Eb7/uWXvzJcMj+FZFGmsszGRNhdhQ/gndxpzgRCM/8l5+X5b5mbSi9dazmy/77URWcuxAefJZqGoKi/XMOqH95KaR9A2H8PGsX3kf/jiMmp+6z5o76ptuHQezht/urPj35sScQbFOW2+F5K11T1Wd0NkcvivwYu/e0yPWn78fycqXW+kVOr5PMFkLBq1s2vBUFhLa1g+nBQJ010n6GR5LEaPbP9I72mPy53uDlA+Hl+db0RdezRmukUpf9GiCd/FLWE3QDxXurYyGG8JdxbpRO2M0VN95ktGgV3zC8rqZbwQZo6H6rpkaT1nfSKDzG80rlLfSabekAB8xm7tHWjRtXb6TN4PO9VGOLaLjG25nbtD1fCIP3frymd/zeT3tCRaY2Wi57jR6XxmVEzARelvpeaNxH6X+ES7FEPgvneV8Vqp7ieH7Mzf7pK8+RZ237j2+YPklhstHLNx0EZ+53p+P71xNGkWS3ihjW5bH5SuT3seW4z5Jmc8smOvKiD2nb7dxx1OGh1if95/kp2eC/2blyGaTs4c8G/Hs2zAOePPTfjdq281bLP+NSb12K+rEin7OrqT3KUtXt3FyN+j7Fs9ijHyEHqiHdSML9rC+hDuV/T+J8xbfK+iwZfq635eujb5DXWbj/x5fumT1ynEHrbS2vrKosDi0Mt/K5b+/gwEzF0Nqul5PxVg4ZHKOdbFjuccS93LKNJlvB0Ty5znXo7f9+GbA85Tn4HQv8iZJFH1dE7eMhtugvsrWVHw3+KCmFcC19ZDjWSzS5J1I3ltwzj5ovmb9YK0QqlUiWSKda9tqCFEuvUsxeF/hXNp8wvou4pzJ+V/PuZ6h1Zsk+p5EvhN9kifX+xOOFcutFybPGszkTtvr323cvRjEg3DrCcDcaEZ0WGyl1FjM1Pkxjyeq71z4nWz15VCzKYetPjWrvRTNzeGOm6KtdLwCfSXvQdbVP9x18te4abmNWCr3Kq2CyJcnn6HNu4eGr+hxG3fKn07OGXIz32CgcbR+5O78WeJjYNj/oNGLjTDEqmxbb03KGfoPwj1Pg3g9RsEVQVXk2yzTjDXACMQFCpO7Ib8/+Vyo70sEw8XdEX1FkLAvsqLjYDr5ofpWhZaSJu2TKc/jgvJM5J68qKw0zg8yGnDllJzB72MEfWSlZZ6qOqW79u2kuzxSkKbhf0nT3PUFrOY45Gnq/jT7f0zKHnoDy0vfh3uO66Z1+j232XHUrwGdwzkq73o7Vx36ma1X/dGSPN/k8cRSjTK0XP67Vi5sHPvyIOGimY0HYDydFl8hk0MaDRlyXmHaXkfqvDg93X4to33kYvRxjQLEZRu2HUErD9ZXkS5lSOXXtAIYvWpN578P35ToEc1ovAPu3VY2a3Rsvx4TW2AwvMj3Si7kPO7Mms5DuB6f16e2WSDqJK65o1nRtF2L1Rs6cG6b++VmKer7uT5X9ciZu3N6E6cd19le/btknaFPtnMND+MLqe34z+1I/Bwn15aRnnJiRhqq75R8S9L+PIAl1ZdNjaU8g5w0ca0JrK2xXE1G1aqB9Kb2S/lr3cOjViSHUYAlUc8a03pN3l2jFp2+krusJqxU9xoF4C7PU4MdvMlY5BcMA2KD6zk/a9/Oi+Z6aU6k6Ji3aQVDRgLmOjS6/rH4tiPr/Hcin4UB/4xMO9KW/eAQvQ4viC2ni0NbyvuLOvxcJ39Ouu1cw+TK+ynLVtzdRWiMX1QEie1Eil5VZGTgKy+a+28eIXTNt1zVpxWPFFjUJyYrXl9w8uqYi7tL6hyrU+Eha6PNhyeIrw5+96LyFkbMjEbdXePh/I06+nZF4fLWyb+B66bzgSNc3qbXJyOWe5trO4/n57nLrXmM2DibLYbVl9CvQ413mOSZKqJHE0Xns8YL5VkT3sw5NXYCGQXLxnBo6RTkHoiR8G18eWlr0sxBn/ftOu5DNy1yFKd3X66TF/xriWtNhuzpheW2MZKtJ2Z263iKtQ5f2/qJZcePXb/eezGzoZ22NLc5o3FjJvNhrHuTuChYUlUWbASSmrBJLKYBfx5AGNSRRyW+hn3CUBlTh7gGVlsN09Mi17RYs2ELfC6mweu/ommDy+iQT6Yz/bvjWjfTgHV2Lfv8oM5o6LABNoljs7J+sWK38L0jhV/GXObva8sXJ1fTcL7J3VpbH6ZL7L5l+qqcYLiY27aK0lJM4m0RcfOXMZz/BI9LdkqzosNJY1caZR49cDQuLj26nLHvCljuNO74386zor0J2xVvHj0UyWZ1wpj5S514LKLyTvPLqm3UdnbnGxpfFaWCw+UzyGzeSwyX4UW+jIWzNSekUNDB2nV23jHpac62ZPg0Iw1PBJbo1kjGRD9sPdw2oM59QMZrrQijRxiQcWFULOZyYjdPa3zv2Nbz+D6ElYlRwSfW3ZiRUejvrtwUzmuKQdtcc4Fi84Fc+3su5CkyMBj56o4V8R7X3VBGJH7Qx9k2xUsdlzEaqvdchG1IP2z1qd6zXwdSp3E6Kb8gOn5d48Zp6Z4znQ5xMZP+cnnMwEeZ+KBOJ+erjQ0apjF0cDnVyaholWhAh+krgppo6HmRa9gfF0zDcyMv09AeEfu8NAd4rnsaox4frNior18nCB8S4i7sFPkyBHwFneuCl3KG/4ABsTUN8Zdy61PFNLw8F/5rWRume43xbxR1I+9qoiFfsjyZ/Z2ilpfOtnwS8V6hvIf4SxL36zZuaMRzP2m8bh1tqZeHmbHlcd2ztnejDs+2rYO4+zxcCfftMmZwxPM+tZoxUTRBPC/9Pw29jL+P/3rQH5GC/KkcXo/uZaTLSNIS7vKrr3IoFdfNyopUUgAja7O5DnpoEq/KpTksXIN78+XMOZizM3kMV3Rz5dr2HoGyz+Y6mcGjrfjn1d3v+Eql17dzVifmOuiR1/0YmofjdQkX1LBAvJRxGqOhek9F2DrZsNWnes9+HUidbyM8yU37nPyIOwsDYj7P5X/KXd/sUc/JfIKOa0P+PHdGJG/ta0zYGkt15unTwtwJLcb9vV89DIEvLS8Su5sqiNgbY/MG4ge54Z/NFLEsJ2NDNh3qpxkdIg/rEHfUn1lOZM2U2YMW8Fx4EI8t7mPodxHPdU9nhOMYhmbVaSbKH0wWPCIWjm8QRL3IQAWw7cjf2ZzG44lP+FzwfZbnjqDs+qw7+XhfuhFvldwT5p7yK+lfF7Hdt/nM9sc08l2Yf3aX7vz4QucG7vg0MhgTRi5+xmqZX7Rv2197dtqf+iwxdRqAzu5QOWjcL+BxwzGaGJe7MarRhl8jrvviy7MH6Y2N/oxq3K5wGGIXUdJjxn8yaAOdxI+8yVH0yMQuKGCyqLU/jzIWu2npb1OOGyfOHPItaelZ/nTYdNfKTj0TPZqQAZZSoq9L8rDudSdv3Tt8Ev3aFsuXvsF5m9Uje/4022nwX/5H+3I+H49/Qv3CosJ7+gqsfQ/X321cr3cxcvWQHXWyvYLMn/A/MxaHT6xzUV/GcFVKTjr/qzVfVDvjSIIGdGe2HLaFMMwD0BC2OotWsPlQLh5GjAaCGuBO/mbmgTdh0uE1Qf/KuLUQFHf07zDJcIfKxK+jcfQc/2V4po6Wv6rFVv/0IxwGMqJqXDQClh4p+PWlb4Z/p8wT93mMdyTGQydGFub2nDn3rRHWiFi7OLBXVvP8vKgeLWG52h9wvC2fS39buxp549XgY3gDJz/NtV/25y5oUq+T5xzPnXxLPhH/UXGvJyu+kfBr4H2qqIs+LMLwmtUjLJUx9ag+Dcho0GtkychBRgMTyn5ORlp1JA29xqfRGxno9VU0rD+vvlY+VettHk9U/5kJ25B+2OpT/VdAfc0hzX3Gc73HklH9zMa643RijyOSkV4dSONIyqhJk8GJmnWg2Ektoh5NaKTFiNFAvdLAcdT2jRDV+DTqMjpE9TFVMRpIRQ2MolDMg6jXIqPp4HqtAVP5eqkBzQPQMGNYRnWYnW4trpdn0lTaaKBmNJBGNn/A9jWTXUrmorovg7+8bZKSpa1HhQpLR5bKp0wTIZdAt1QuZAXKplnfmbBjBeKYoEYDRgPl18AhBNWEY7Ub9VX05ogWtYrWVwWkar2N0VAzZyZs8wA+Rm0H1IzqTC5GA/VOA/V9QSed8BPAzGeod5e+qbCvgbDNA7iKisXet/craLZGA0YDSdGAXjPknX2+eFl/RQsmaW2N5vVXBabm9V0DYZsHoFXqZtT3k2rqbzRQDRrQSoKzqiHdupSkRhliaxrUpULXl7KaxxM1c6bDNg/gK9SmO6GmNaM+k4vRQL3RgB5NTKo3tS2+ouZVy+L1YnzrmQYmUN8hIarze9Tl8BDVx1TFaCAVNKClq3ulQkFqqQx6PPML7FJL+Ztsy9CAGWkoQ0FJPKzJkPsnMb3aTipskztrW58mf6OBTqhAbybV50d/e1H/P4EPkRlJRQ0Yo6HmzkrYOtmw1afmrgSTk9FA8RoYgLd5NGHemij+6jC+9U4DGdR4DYRlHkBL6qJFq8ziKyjBiNFAEjSgFRC1RkN9lG3ilf6GbZhGZOvjuTR1TqIGwjYPQLO8eyZRPyYpo4H6qoGdqfjvUF+N8H9R953iOjAj4CgiVaW+XqC1dT70xsF2IOMhDNKdSugZ7OdhqIypg9FALWrgdPLWyN3kWixDbWatieKqv+YzTKrNgpi8S9eAsehK10+yj4ZtHkDY6pPs823SMxoorwb0quXE8gYOYTgt6HQlbBHCupkqGQ1UWgNhmwewK5r4sdLaMBGNBowGpIHWsAI0aldfxaPi4rb6qgBTb6OBkjQQtnkAv1JRPYs0YjRgNFA5DZxFtLGVixqaWDIYZoIZ/U7xU2pOUM2foLAN6YetPjV/RZgc67sG6vujCZ3/9XAiuNoxkroaMBMha/7ctCLLQ+Glms+6WnLUd+81IfK1akndJGo0EH4NaKRuNOSFv6ol1nAdR/QpbCMprgEt2WmkZjWgeQDTICxD+ntTl8ehJxgxGihRA40aNVrGQTPRrUQNmQNGA+XSwPL169dvWa6QJlBoNBCmeQDpnJXV0Cw0Z8dUpFo0gNGg59ZGjAaMBqqggdr+H5k5DVU4eVWIGqZ5APnoQWvl63PZRowGjAaMBowGQqwBYzTUzskNk9EgDYatPrVzVZhcjQaMBowGUlwDxmionRMUtk62yvXp13Xs4/06Z/UIno5+XcYeiX/RBMu+XcZc3a/LuEHBMEE3YR+BFf26jvlH0L+87oG9spr36zbmsPKGT1a4/h0mbNG329gPVfYB3caaEZtkKdakYzRgNJB0DRijIekqLVeCXxNKEyLDMg/gE+qiT9qmQYXlhK7jDuZhd+NJswd940dWR2rb1lPsa9ntmORtbP6IZXvXDuyc1cT387dHdnu2Me4L3TSv86Scobf5/hXZ5ud6Z1OOkyoSJxlhvYz8I23Pc9I7OFtOyB7yaTLSNGkYDRgNGA1UhwYq1chXR0HqWZrBeQBvhqDuWjN+MejVS81vqJA4nneD5Xj3BiN5GXkjPct+ic70EN//9QXH5HI3/nZ+xD0bvwd9fxaSsxt7Wf/wLHrefOecAR2fG/nn1jv80WL5bydblr2HZVtzW6ze8PSoRWdsVJzju4/unOY5/TzX2t6zrZl5G5qNatjoz9au59F52636dRt3YtSOzotEIz0nzRr8vB/Hce3uk3OGju7fZewAwv0edbzjeP3o/Uk5Q17v223MPo7lnEQa6z3XeW7K7EELFO/ErqM7RL3IYOrXlDfQ35w0a8hm57vwuDfYtuxmefOilxLlAUZUdnQc+1TK19qxvLcmzBr6qtLCv78Tifzhuu6xEc/70PfXMSNGA0YDRgM1oQEz0lATWi4+jyoP6RefbK35Vqo+x/cavSWd+t7p7SPT/ZL37zLuTBaUXe149gTfz996njeZY8P8/cKtzQCB+xNuz3asRREnM7fl8qVjLNs+niPTidNzVbOGPObw7L5ds7pGXOcdy7WXe473Lp3+WZmNVt8RiWTkUo4VjG6swSBYluY6nRjVOMXPR/t07MO1j6Fxout4zxM2nbSbycggzmhslm/o5JdGHHf6Cd2y2h/d7vlmBZYzjbL9jEHzKek/2q/ruH5+mtrmuWnr8V9GBdY7lv3zgI4v7Gzb9heea6dZjv2Za9t3E+cmhbUdu7/nuTEjxrUcjawYMRowGjAaqFENGKOhRtW9WWaV6mQ3SyG1dipVn7R8a0+64Xnjxw+KqjrMY9jVs73LczcUXFZc9TK8SDb+e5zb6/H04PE0NzKGfXdi9uBnCiLudp7l9kmPOkMnzhoygdGBcz3P2m5A56z9Hdv1bNsdNnHW4McmZw990ba8ZzBCOo7/etAfGBJfuJY3f2LOoPeCaRfnpmN/cVL2kKsnzxo6DmPhZhK9fGLOkBcm5gz9j+fZTziue2l6ZsZWGCUZxP++Z/b8CRErekS6a2/2+OHl2YN+JP6HhFlE/PHRtMj5lGcCIxy3TMoePAYjiFXyvOuPbvdapsqBcUJ9hlzP8SztGzEaMBowGqhJDSTFaNAEshO7vrCLX3A9j9ad1sCBWRHf74SeY7br3+3Zrfz9VNiqTKe3ebpBRcqiuvJMXas6VlWqNA+gqplXQ/xKGQ2W57DYj72iqDy29Syd5PjMBpE9PNvtgX8TzXnwj4+fPWgtbve33KYlngMea7QnzfR8x32DxxnThOK4EWub5qs3LPC8yP74fQo/EO4Cwpf5P8AQoP8PiGt9H9jbnY7/hqK87NhjiyZ6REGsuxnxeOGbrh2WRa3IP/Ls6GbGTiCNmBMjZhdGNGQYxWTizCHf4rAzm6zattDHDeZb6GV+jQaMBowGakgDZTaW5SlHfr77SNRNj03qY3b79V563ueO5z6QP8+d07dzVielQYvN4+mMUXQImze+5cmgmsI4UfvVlU0aVGi2ekGeewGd0YgkFCk4DyAJydV6Egspgc5tm4qUhOH31UTaNLHRs9aQzKF0tDeTzumwNRdpbHhe6cbvuNNarc1dpf3ixZVu/2CuQR8fzIJ+663cqSuaNCQt7zDXdU7m2C6ubT3IhfmXa9LFk3Kk+enzKGPzSau2FRsZiR9fyRDA5X5eBY7Tz3LyrtGETeYlTMB/e9u1+jCC0JQRiof9NIvbYjCsIK0i41qPOAiXkbs2+mcsvOcE8y0uCeNnNGA0YDRQbRqostHA5Kz9aQybaeb7CZ2z2tHgXZ2e4exBQ3kM9sFIx/FuUelfmTloCZsF/buOq/HZ6dWmvaonXLm786rnW10pVLg+aU7BPO70d/MLxETBv/mdr+05l+P//aScwUf6xxs0Wq2wi/xJjb5/cNti9UbKYWf27zLmUo129esypjed9xcN3fRWTEjUMsaLNArAhMltmfNwPgaFHiEQxctl5KPNwJ5ZrSPML2D0oNvxnbN2ovPfhnBnx8IU88MgxDgeG9yqESjN0Uhz3cmWmz4k1y7YBePnA43CTZw9OBuTilECTyMlJQqPZsajj3NO6DK628D9shpmNki7m8Bvvr7g5NUlRjIHjAaMBowGakgDRXdSlc3PsZ0rXM/V82Rrj9lzfsju1q77+BknF90F0ggWGSZMGhtLA3s/QccH81PDzqjEfRgfx3LPF2VC2OTuOXNvGGGNcBm5OJmG/iqOtSAOE9XcGyZmD5uq9/U9x21HY90H/3bkMxX3FNz3EDaTYeEb9YwYo2awbTudLc/tQwPegob7s2h640tfnnH8+mAZ+nfN2o/n4CpDa8r4PTeaF9OBfa+y5c1zb+dWdDDhl1G+HwizNBi3Cm51ssdDqXefVUi/pqOqPgfAC+XN+KVvhn/HsP5qvUXwUs6weZvFcxkxiFhfBv24/++N/nWeN5Pcgsx8J2PD+/KUQcEI11FMJLyX6+oKz7GXOJbbf1LOsJ8wAu7CkH28f9exs1zP+hlj4E6shZNjI2DeuNcxHIbnF3i3Y7icx4TMkUxqfDePUQseGzxmW06HwkztuRgUv/oFaLl6/bWrmja6M992P43kOQWkNYa5FI/ICunbdewtrhWZwmTGxrr2XNf7y1wN2/Z+442L2UpvUvbQd1iz4caI5TyXv9ZtxnX3dp4VLZyAydwPJmD+4udrtkYDRgNGAzWtAdqkykvv3tPSWiz//U87P6PtxHkDlvspqQOgobyN8d0DbMfrHX8uy2HPpvFcmZ7mtItNPItHoNM4GudN6a7Th8Y/M5Kx4RU+q35d1Lb+ZELZh1HH2fvl7EELaYDPo8CX0aB3wmi4ivSucx3nwEhe2jIvPXc+jfpH6+zcYQ2s9MMcz/kPd6g7YjRczZDwDTTKB65q3frb2Kx67l4n5gy+rl+3sV9brncFk9/mYvzkEH+gJsHRaJ9KZ3Jtj5x5Xb/p0uEizJ7BGenOsRvzrVYRy52mTosyXOrXtwrbtsT9AHaoQhqpFHVPCvMk6NXLcgud8/kYgDvRYd5QViSugU84l6f4rzSWFd4cTx0NaM18PrRT3jZH19A+kA6z4D2oKdH/8TDQf5N5L0kR/Tc6x1PSTZUeN8kg3uzmJX48GZtDSWQxMLpVq6LzfSo8D2U9WjucMMxHir223ZJtsr56uVM83bfYJsoueKj/yYHYTUdigCrsO8R1QfORegH9WtWlgv+jqmeYkIIqVWlp9fsv2xM5GjQYlFiuk7bG8qIjuVo+5c5QIwtxoSu2rIW5BV5H36dw6+jOvUue495iZWzsmdbBOZTO+5Mp2YPmp3tO2zRr4zpGAg6hs96RuzeNOBSK7b2lMIX52/PJ64U3s09dlxmJfMIrblv6wRiFGDd59qA506f3KeCu7n6O9fOPaRuxIsey+cXzoraGsplg/yP7237dtVNnRif6YVg8Mn7GoFUyXKjBU4qTJFlIOvpTtUlSerWdzDcUoC00r0hB0jz7KUaJDihrgqlWiOR8fG4Mhopot06GvZlS61rS11P/DdNhLNSUyGAZBfslMcMhpMWoVmyezmVsH4AlcBRUhyiP/asj4QqmGSH8KJDxV5oczMFxsA3sDmdAskQd9tUlJDYd/z2hUQnHK+vdm4gPxSPvwvbcuLvOb6pkNHiRiBSdm6gFzV+YOGvYW83XrB9GR39EbK6DH8jzciN2dLMTNCln0AweXWDteVvwnvtzBfPcxf26ZvU6tusLLZl0+DITKKczk/4sJUGHr042JriKHoMwXOwyBK1vslsbnQIZJ0XCE5I//B2eba/iYHN/X1vXcluxaU6HdFoRljU5zSrIJ8cWsclpfgTPloGTTPmIxA5IZoK1mBZD87G7pwo1trwRkZfvOMMKIhtLfVzGqNJiN73R9bVYP5N19WtAbzNpxOld0CTQpvAm6PFg4qTlogmjHEsUXUuZAc/GAXeis7RjwbAqT0nSsKQDAf/JuPvEUUd2Oaij1F11UErLJxgu6C5PnM3avWBk3Ju1yYFj8i/pf6lzVZqUlKbiJJZ3V/x0zv8Bz8IgSJRNE6Y3P5KYln9UhkpJZVcYXR/ciFrnw1Qoj5RUBumiqG/C3QF8fauNPwGCUppeMwgoUlKqZDRYBRlLqVUr5h7E0mGlvON4VvyeX9N1jRtLMRE3zdrg+9Epbxm17d+L9nHEX6nbWu/TM+y/M5YBw1Luaele5GQ6bVuPI3gn/lTHcT/krARPTDCZkt2e18k/yPPtHhgBc/x9bZnzsIDNBh5HnEVeZ2yVsepcDIsfHTdN9WOintdN4SRYJh0LXUn7DZPRIKV8CBW+w4kZmtmnSt8likaVEueilBjYHKirGnApuIxP3W32BTX8Z4KGrbNBIqNiPagd0Y3CzSC5CHTDoNHN1XEUVp31WlD4fUDyIvwAz8AaWAHDoDjpj+dvsAoWweEgGQrK7wVQeW6FiojyVhn8DuUw3N+C2oTv4RiQfAW+W/u3wb/kQJTnEvgYPoDtIFGU/kL4HBbBAJBo+x58DYqbA7uBZAvQSqQzQWV5GGLtPFvp6yVQnX194iySQbh+BKU7usi30HETG7+8KvNO8De4B44GtR/HwfsgeRaegO9gEUwDv+M+EPdckL6kxxNBoj7iEVAZdPxMKE6y457z2Gq0QTfAW8X9tNF+KzgCpoN0NQt0LRwCEh2fAnNA+akeulZvB103/wOd1y9A0hreAD/8k7gzQLIMdF5VnqVwJ6ScOFUpUfyxxMLYMD4JrXPypvEP2po5Cg+zSt7QvIgrZT4Vf3PC6tdjYgv2W26TtlqKL5KI661i3sGjzDG4jPkEZzOCcDCvvL3tRexvmUneHkNkoOYZsEjwCCI18Y2UogTKcJDesTwLv6t/1zEXMcHxXst2NzsZW2Ws1J9j7Tdd2z/fv9u40/7IazGOq25f7oD/xEogvH0VZbiyPxPUeHRRUsNSRilKPKwLPiwjDarkx6A/sxGjgcpoII9I18K28ByoIR0Lu8N62BJOgpGwJ3wG6ogywBc12n3gJ7gD1ECrI1LcK8GXtjiUnzqtX+Ap2A6CsgM7o0GPS/YHNejjQSMgvvTCcSnIv6Iyiwiqm9rGcXAydIXj4RlQpzQK5C+haYq5n2YrPciw6gBdQO3tYxCUrdh5FgaB9KKOTHF3BonKPji+/R/b/4DkIZgPu4HKp/RPB0kjULvVHGZAULZg53GQvpXfdPBFdRoGHUHp6bzKIJgKOuevwIGQKLviIZ3ommgJSkdleAnOBx2TIfco6PwNBLWp7UB5lSTd4wdUxy9LChT335ftBdAGpBuVVyLj4DdQGaUnGWg6lzfBRDgbgqK4MoB07e0CGum4Dnz5E4f894C/g+qbUlIlo0E1YbmZ5iUAAEAASURBVB4Bf5ToMXJrPkG+FeUtBG8hB/Z0POshJiPqpBaKm3c0MaaMnHFevu+l7cTZQ2ZixB7LfIHmzFLfwXacs6ZkD50yeebgN+iwz2VOQmeONSPto3kr4uzZvTo1jUaiUx0voj9Dodj2nZYTJR1uU1a10FLA58SPsPGe45+2EKNjSx5iHKcZ6jqGQfJ/jGbMU3miGY17s/8eb1C05UnHq7kbmunkW3qVNI3VBalTY+Y2MFfDOYLyvKBjSRI1RrqA9AcMg3xCJfaEtFSszPHdstr27Tru8IqVzbNlTAYXK6tY/NJDa77OiT1eUMNlpFAD/2ajBliNqf6r+8BkOASWgTrvN6E/6E5V11oT8EV3mZ/B13GP29jqxkANcsu4nzYuXAZK6x7IBOURFF0rDeBbUJnUkapTUKfky79wKM8c36MC2/R42IPY5oI6udNgL1gD6kTV3tB2xgyVg9kuB+V1HCyCAaA4qs+RQHNXJEpX7aJ/pyudfAyHgeRdUN0kz0AfkD6V9mpQukPgF/gb+DIOh8qnDk5hhM6H9DIXskEiQ8Rv7/3yKpzCSw4FXwcxj2J+puC3EZTOLFBbuTeonG1Aaamey6A3SAdZsA4KQGWoqswmgTnxRL5iqzJIjgCl78EG6AC69koSle0RUHjV6TGI9Z9sJSq35AdYBU21k0oipVdJeKvgofyo+yYN6n1aCvjVnOErSPCBTYnqeouL551FZ36Jvxvcal4D+2IzmZw9+GU8hC9j4w4ptEh4rPC6v6MPG+F+3t8nzwJenxzp7/vbeNqx3fiw9+P+seA2/iqgGp7qEF3UX8J+MLU6MqjhNHVeFkIPUL1SSiKWty//15Mp1NvlLdjAgeOd/HneKF7Q1B9aDUNShVeWz/cKHJXnu6QmXDcT60Kxr4Dn4J9x1NC+AceDOjg1yupAXgR1ULuCGmFf1FlI9N8S67WDyB3sUNXJ5ukAog5QIsMhKE3iO+3ZNo27R7FdCS3j+7/Ft5XZ9CSS2rfG4JfTT+cWHOokl4OuDxkH6hyfBklinGX4nQeODsalIdvEdLXvd9Rr/YBspQv1CUJp+3rEab0FMhx88fUlvZwe9/yJ7SuwMb6vjdJUeyxJTFNtxdkQPCcKlyjB9HTOJDovwfLJT9eL+hB1wsE4flnxrpAE9RhMzy+DEks8B2Vl0IgAwfMRPBeK6+tKbl3TZelG4WpUdHFUSfTqJI8Onsib755BQv8rKbH+XUZjkdlfsua//uQ1Jjz2wFr29KdLZfmQwu0PU1O5kBUo20eEPQC+LCuOJrumR5zm0UiTpcH5CiOYJzOjW6edI07Um/TN0EWJ6fAq7Y5OQeZ6PSLjrYuM/PTodolpKE6/HmPapOdHVsceNSUmEt/XR6IKIvbGKbOG/R4MUvg4baPuKtUYlirHdc/a3okWFCSmoYWiNkatJr2y5yymTkWNjb6dsTTabHuV2crdvO3T0uYrWzTYRqtABhd18uPYbv76iWXM/yi1sKl9UI3z6dALTgF1VEeB5GegHYkN6Z7IdkIcNpUStX8nwyjoC5KZsF3MVfjzTdz9BdubYDjsCTKM24KkoHBjbc/2cND1vyDuV9rmbA62gpdhR9gq7taIiMr2P3gfJDIULoPu8HeQZIP09Ix2kD3gTPD35acwGgnxO+xM3PvE/bZmq7pEIAoHwizQOVC8P8BP6yLcKl+iqOzCl91wdAM/v864fcNLafYBP82uuC8O7OMst+QQcgt4E34FdfCPw+fwFaj9eRAkqm95RPVuBUuhI+gclCXSl9LXVjIZdK5K6vClg4PhBZDI7ceNeaT6T3mUUmYdJmcP+W9ZgfQ2BWFEjQojECNrNMPKZfYx0fyGoHIppFYsNZpqhB8qrViaw8Jf62Kaq+/Souu6sfbGlfoQ04DO4w74hq9IRjx3kRW1t2COTAZrgRwgA0ELQZHmO7AFy5V345PU9+R7bj/CuZHo+u35nsg+U74e+gvh1hPmY5rz9XxKuxtGxr/5uNR9wfJotUcWb3qFdTr+ZEGxLXilc/5WmauG6nGV5r9Y0dzrMXS/z58XpQGwg1GL3OSziom173iuuzWfsNiVRx8vTs4ZfLHWMGm+/PdR+QUuH8myfuHbE9ud4Iw5ccrMoV8zYbjj0jzrFeba/BqJrmsZS9orHPno223c8Ss97z/oZG5mo7T2pH8f1/C/9fhiaW7kVS08xuP7ndHdXD5odRLlUuMUJlFnewuMADWwvnyG40nYBjbAY3AzNAPJjoWbCv1GCa28HoamoIb8awgaDR+yPwZuhJNhB5gK6lh88c9BFzxGwemgeiSK/hPqjCTbgvLkHMZGOeawVf4fwHhQZ6J6LgLJG/AETIc/QfIf0H/tRZgLp8FtEBTpUJ262pgJcDxMA8UbAK1BaX8Bii+jQ6L2SPWWkaJ+QuXsDWXJdwR4Ht6H10CPNPJA8jicApNA5ToV7oXKyGIi/RumwzjYGzJA+f8Ep8No+ANUZ/mXJa8S4FmQISL9L4eyZAQBJkInaAm6DhX/UDgargPp1pfrcWTBnqDzfyQobJ0Rp86UNNwF/YTq6SLSnzMMogbpgDIr4lmXcIMwhE7xGCa+nshrtW0Vh1UPh9AVXoW/vh3RDa/VXlr+YfH0GmtlUfwPptO9gUdPf8egOJr9fWm5Z9lR+5h4uAY2Fj+PpU7ga5f7MhfmOo1OxI/FNhHb/Tcv54yZlDO0N2/OdGdUKn1pbvPztbw081vusAuiB7Cq6CEYBWqcS5JMbipeIv8DnILoPszJuUDLP7Po2ZkYA22YG9MRo/pAwtxqu3w+G6GcD3DsEflT9oPZbyX/E9pPbspE3Wdt1zqe9I6y8zJ64X2TvvwZLUhTo/0WBsnRK7fYuhtPr38b2HP8looXQrmVOunZ8OlxdO51Pa0BNf5qoK+BK6AH9IGloM5I7hyQ3AaHx1yFPyeyuTqw7+LuCReArhsZBZJPQem8rR1kGGj/ZlBHoE6I0xYzXuX/GUjUOWhfnXCiPIrHcLgljvJqA++DLxfiOBsWwp1wHCgfiQwcdTBXaScuMh5UfxkbC0B1fQIkN4I6L8lZwH8tZoBcyVYdty8quzqyeXAAyCCSvAPSzQz4EKTz+SBRJ7cu5ir+53K8pWeVSYaSypUHq6AXPA3fg3Tp/7dkYOi8S9QeXhpzFephStytjfTi71+LW3pUWg+C8imAtSAjQteDyr4f6HpJFJVJ5ysaPyAj5g6YA/3gWFgNX8LF4Etw/2M8paevQeXaF9bDK6CyfQ5fwfkgmQY9QHp9CzqD9CSRrn6LuQp/EvcDh2rPmVZ7WZucAxrQn0kNhS4mXZB1XRZRATV2bUH1KkleY3nvqdxNv8JE01fzNjR/UAHpMC+hg99f347Q2zOkRDounXOhZKRH3pWLOD/b/MHjb/EwPun9yjocDePBoq0zVmXJzaOJ3+h4v3Qc6wAVqkhs6zAMlSWMcNz8TczTS2dzSDSSxvLSXs6EucMXy5vFxiawHLUamGIl6rlv64DCU5cCmpmGrCJ6MGUbH59fY22VsfqFpXnNn5BBwkeyDnbyo+cpjsrOyMObKlckc91eTNZ1GE3oR5n6eYXzx9YymHCQY1vTWPb6LdJvby1b+jp1vzu4qqrSCpl8S31EcbIIT+HLdN/BdknArcY/KGrgE0UdrzrdoCxnZ3rQo5h9HV4aR26J0pouRzGijk2UJeowRXEyuxjPXPwmFuM/K8FPxknQQAkelrEjEkW6fD7Rk/33ivFL9JoW8Pgl4Nb/aHJg33eqs/Q7TOlfSGTMBCVxv6SybyRS7P8fj/x7MJG4W0bj9IC/DI5g2WRQSVbEie0Us6/6FaenqX4EtjISfClJrx/5AeLbxP2Ew7Wz69ROtsnLlVchh9MhXJK8FGstJV0gsvTDImXWB+PgUt6wOYZOcTEjBjdnNlz9gSrP0Puz3PU/wj86w7bcSdyVzwwqZe2aPDWUeNO/Wna+f4yONGgTuCNnnKtGoEgwQOjHN5N0LI9FfLCqED48RYf9GNYH36squvuwNMGXWJulFUwlUtAgeMwriKZRfEaNvE1prNilpRooL+JkqoyOle75dzcWoyyx+rh2JJ2D64vKo3LZ9m1M3vx0QvaQT5kwyTdWrLGkfQD+cwd0HrtXsBzGXSENqFN7tkIxwhdYRrHudo0YDZRbA3XeaOCrxjuxYFO7ctc4dQOW2cmmbtGLLVmp9dEzf+6ap3mRvMW8AjuC75EcSiq9NLRPZztE35fQHIS0pmkfYh60pxOPFJtLyZ7p/buOP1iHNdkSw2RPJ7/g44TgX2CsMKlw8DPC9bxdMGLaMfTPHYbdXV+sVPgB3cbuy6ZRQtxSd3m88jkBjsFOiBkqjFQcyf5v42cP/J1O/2PPTde+pcmN5HWQ3F40+g2BmztOwccqz8b1TcdiCJ0UddMaaJ0QFiHrxyON//HdlAGaR+FGYo+0FNVIxTWgYfGzKx4tVDF093t/qGpkKlPtGkir9hxKyUArSDIJbabt2CfSODbyrMhLWvVPUY7v9XKjtLz1w7l33I7nxDm8dcEQHE0xwhcMO/EhLBpOrfDmcXcWa5d1yOrfPetAL+odxh3qijzbfS7+CqjFwkxd6IyOUxjXcl6dnDPIf+4pr1QQdbL3pEJBklQG1afERlnfAeFDVR97VsYHGA8fM2GwG+fnkfGfDNrA/iTbcUexINc7eevcgzi7P3JnvX1Fy8WaG49wjX3NRcMIjv3ghLlDF7Po2P5+OhgJl/Ghsil8rl3GRQZzJTrmR+w+r8wcuqRv1zF388XKz/t1HfM+jwW6kIaGO8st6+y8xxtaGQP4QBtGz9hFRDwM4/a02DUcHcuXN63XSfvwpbnWrhgRMeNdb14wynJ9NJr2Hjp4g8ep3Yj3XY/Zc7Jndu7gMQLB44kxzLFwuOa9DraVd065C2QCGg0YDRgNJEEDtTrSwD3Y/2EwTPFcj/WT7BaO536mjxYd3e61zEjeug9p9PvQxf9MuKtoRP+n+jJbfh8Mhml0MJpZvCuN+ZW+HmhwL2AW++Oe4y1hmHn7dCvyycBeWc01kYzwU7npW8kINp/Zi06T4eHHS5HtIspBday2KVKeqhZDjxR2hhYlJYQheGPUcQdwTp5heyqTFi9XWBYEG0zHeJljOVM3WHlHuXkN/5YWcUbqGJMNd3l9wfA1cmc0tt9Oc53Bckvs/My/u/kNR8V2tG/nHU7n+nQkEmWi5ODb5b/eyp2C/xlyT5k19Mv1du5uPCd4hMmH9/DmRGd/9VKWNP8n8Y7ijIzi66u9XddpP/6TgX8xHChP++7zsvXMMyb+vhY6Yx2QgxyX7wt43hPpGU772GJlhNJiZukZdkfSHslkzKFMhtyfRcbGKQGWS3/IsfP3lk54HHEG+0NG8KpmLI7Lp7mJQ1kfbLFmQ/cQv3YZ06X5MRowGkg9DXATV3uCIcAQrvdfZq+PUim4s5zrOM45bjS6PS3+5ZNzhuwnf80qdzI2LOFOa09GHm7ljvAzGvUHdIw7rye5BVub0d6+kiHgpTSohxauMBk7NpqxiU/d2Lcl+Lql6xyvLyTqY1jprrVEE+SURgqJOo6X4fkUKlNVivI2kTX8+XpVEqlMXK4tF2Oj+ZT5fWMGRmXSMHGSq4Ha/qRvGbV5luP6/+kZv0Y7O8JfjET8qlt4XBWbAFob+euR2SlxqrueL5BBX3gM1OYth8TJm3glRb4jlT2hO1wB/SEZciCJ6O2KbPgX6EbpGahWqe3/Ua2ONEizfKr6R1/DWDBreK7LLDJrdzr7r3z/eMM/3456u+HX3vG8oouLOLFw0fl523KslRexJtNhLBSkfrRm32+dsfItnlV/yAevshl1+JZlovuvimxMxc7kI+rAUHpopPbq49m3ZKbl5oZGk6Yi1a2BrcmgEeTDGaBZ/rUhNGlWG6iNtln1lx6qWxqQwVDQSOTVcBdsA9Ulykf6zAG9hpss0ejlDvHEtmTbNFkJp3I6abVdOM9lbCBRWMWREYXExwet+O7DSuY3rGDuQ8tNUezYiYqkZ6xx81w334v29Ocx6FEHq8ZHF+U3zGjuOdc2XrvukpVNGh6MUXJ7Qy9DE+uu35ROSrjUyZ6dEiVJTiFUn1rRMYtE3ZKcKphUakEDh5HnEFBDr1GqF0HSBi6ArWAtaDLjPNDcj52gHehuUq8sToJroTnoblavMutYG9C78bvBZ/A4eOCL8jwEPoQuoHR3gZ6wADRythEU7iLYA+bDXPgJim52cEu2h8ugNXwPD4PuToeBOjEh0b7uVHVXLDkU+oPuwJXn7yDRTcXpkAvjoQ08A4NgRXz7JtuJcA7sDZI3YGzMVfifZJQ3tm7En2yV/q/xYzbb82BfWAT3wnrYBvT4UEbFYngQVoJ0r4m8OheJ0gwPxWkLS+ABUH1uAsnFoLTU8Z4Kv8AcOCGODLinQWWV6HxOg7OAp4qx86q7/ERR/3ANbAtZgYOtcO8BX0Ff2AAngs71c3Ac9APl+wx8CpKGcBm0hx/hX9ALdobhoPPui/Sm+o6Je0TYXg/3QG0ZovGiJGejCz/lxI06r9KxH6X5Cyocjy3OYJPOe3BfMPchi/mQl+mRBUbBNl7hH8gaP2PQKsJMYx7DrfqwkN6Hz3fczwty3cMauZnH4n7z97T0jB6z5r3NKMZcmglddKkmajR0IbZItYJVsjz60+nPlV7J+CZa/dNAG6qsBnsCjAZ1WkeBGm417up8dFwd2csg6QYKmwYvwd0wBRT+W1A4GxROHaduVB6Fk+EGCIrSGAG6ZhX+BWgGSl8dzY0guRMGgI5ngjqnnpAo6jyWwRPQHvzOVZ2NjBJftN/J32F7KTwLBfAOqFzq8FQ/dZxCx08DySBQnTQKq45ZndQxoDCvgPI9ECS3wLWg8Go3lb7qK+kNaoPUiR4GCisZBetA9VAn/yRIGsP2MdfmPw67b4GMLp0v//z5W7xi6z18w3Y1qO37A86EO2AcvAsvguotuRFGwARQHWUIqVMOis7zVGgCqsNFIN1JdoWzY65C42Ak7nmg8yOjRTrLgrfj273ZSqTzTqA6twSVbSFIdznwO/iyAod0rWtCIh0eBaEwGFQhX5ly17jw2tivzFPY6Gfs70/KHrSA5XRP4+w/xmMGWYuzmI/wN+Yg5I2wRoz8ukuHHZyMjXPyY6MO9tuMVfypNJg4NpzJc/9hbsOPViR9HfH/O3HWEC4wvlLYNWvfzIZp37Kcby5BP8zbUKALJNVEDcQXsB+8nmqFq0R51Bh8Dz3h80rEN1Hqnwa2oMpqcDPgPdgL1Eaoc1CH8iZI9F+5KuYq/JnN5r74/odsP4KJoHgyItSJSGTIal9yCUwCdVIliTrh2+MH1Zn+DdQhng/dYTGokzkEipPWeDaFn+FcaA5liUeA0+EXkA76wkEwGB6G0SDRzcXwmKvw51k2D8X31c7pv/cbSKffQVuQblT+80Adn9qbAXAwSOaCb0gp3jnyRFSPZqAyXQzqPCWfxIntBH7UhimO4qs+00F5HA9+2yY/idqJmSCj4Rq4DGRwSHaBC+FsULmvhRzQcblVDnX6vvTEsR0oDeWrdJdCcZKF5wPxA0pTefvXVxvcyldx94WtIR8+Bxlo0t1KyAaF8WU+jjlwHMjYOAWegdBIWm3WhPfN9WcokuB+MV+3jIXDaHCxpW9iR2wm8Y8FnbSZZ2zH9ibmxN6yuPKvx1LOR43dAeD/sVKugBUskF8f/dmMGA2UpYEZBBgBd8BYeAMuh0XQCe4DdRTqDGzwRZ2jL3k41AFJouCCOhzJt4Wb2O8ifneAYDqxA4GfYIcg40XpKH91oIvBF3XQxcnJeKouV4M6lNvhRUiUYBk2cFCdsy/qoFTOneF935NtYp4/Bo4V4B4FneEn2BZ8kX6Upi+qx46gTjB416xy+H3EGbjvBHWIP8Tdo9mWJEpvAXiBAKr/9oH94pxt8XwSVH5fPvQdbP3zoXOqesgoDIp0tAj8fJfjFsWJ9OJLGxyPQTDfT9lXej+DDAaJ8nw+5ir552kOnQy6dv8GF0FoxP8jhaZCIaiI38mGoCqxKoStPmE5L6laD3UqH0AX2B3USI+Ao0Adr2401CFdAsGO1u8k8C5VdMfoyw44foXS4hZ3TJ3rWmgDvqizS5Q0PGRgqOPQXfvD8BzIX51eOvjSynewbQQtAvuqr8r5M7QBXxLzVJq+TMTxEiju/rAQ/PZe+Qbzkx6WgKS4+tr4S28ngOpxB4yCxlCSyGhLNBCUT9AoKS6u4g0H1U0cCndDeUV6Cuabyb4MvOIkqK9lBDgd/Hz74FY9ZXCo7jZItJXx5Nfd99cxX7JwHAyqx5uwGkIj/kUUmgqFoCKybntBegjqoip8BAeEpC6mGtWvAXUsr8JeoPZJndgvEIEoyIhQI+53JBm4KyJHE3gAtAONWjwFFRWV47+guIfDjdAbEqUAj//AhaDONhfUaSr+D6BR0V3hNOgGvqjO98BOcAU0hPfgUZCxNAxUhxFQkkhfG0D6ORX0H1QHKlFHdz/sDBeBDBSlX5KoPPfC5bAlqB7qTLVVuZVGonyARwO4HlSP86A9vA6JonIeCFvB0/BP6ATdQdfC3lBe+ZyAa+A6UL4qd3muEeV7F3SGrvAKyNiaDUvg/0DpXQV9YB2sB4XZBoKi/F+G22EUhEqM0ZB6p1NW6ffQM/WKVqkS/UisfFDjaMRooCwNfEaAf4Aa8BdgIdwK6mxGwRh4Av4H6vS3gx9gGvgi93f+DttnIC++/xHb4+ApUEd5C0jegkWgDn0UqMNPTDe4fyPHJ8FwUOf5ZnzLZjPpz96hoHJrlORY8OA2UMf7NHSAy2ARKP8n4Et4HvYCGSb58BUMht5x/sNWeUtUZ7UbvijcKfA27ADDQB2dL3NwPAcHgNKXfvRfVT18+RnHG/GdgWz3BdVDdT4apKPGsD0kitJTvVU31eMgOARWgOqic+LLv3AcCPuAzseL8F94II50JFE6G2Kuwp/Effm6oLKp89fxhfBvkJ5kfL4Kkg9hXsxV+HM7G9VNYR+Ob0ey9UDXy06g9DqCDDbJQyCDZj+QkRRMbyz7G+EdMGI0UO0a0IWrO4ywiP6Mp4alMqYeldOAFqWpXMykxVIn+lKSUnuKdHQ3LGkEi0EdSHWKyn9RIAN1Wg8G9svrVGffvLyBTbhKaUBGyIhKxSwjUm3/j9LKKJ85XDsa0N3QiSBLOwyi+uiO5tkwVMbUwWgADeiangrZoLvPCaCh8eqUj0n8NdDoRcM4Grkwkjoa0MjLLHCgV+oUy5Qk7BrQUNivIapkT+qiP5KReqyB2r5DQvVq0Fsn8RTobr0zbJHENMtKKkKA9rBLWQFLOb4zx9SpGakeDej8aPSpWiQF/kfVUi+TaNU18CNJ7Fr1ZFIiBTV0K6FlSpTGFKJWNGAau1pRu8k0ZBqo7f+RsTZT94Lyh/RTt4TlL1mUoBq63b/8UUxIowGjAaMBo4FU04AxGlLtjGwqT5iMBtUqbPXZdKaMq7Y1sAcF0Ox+iSYLdoq56taPyt+jikXesorxg9H1uuTT0CTomYLuKyjTVrAtXJqC5QtdkYzRkLqnNGydbNjqk7pXTv0rWRiMhkGctu5VOHXnEPcfVYhfXFTpVY8WU1mupHAyGmTc7JvKBQ1L2YzRkLpnMpui6f3qsMwD+Iy6qBEqz0IrBDNSzzWwI/XvDeoQgtKOnd7QLehZgltvh+0Feiymtw1KEk1oPAT0hk9p4VpxvDdsB3Kro7JBZdVkSOXjd7Kaj9QbgoaAJmIqnOL3huL+25pEdzCUVo5dON4b/PURMnHvBs0gONrQk32lpXx90URQ7ausckuUl+q+J6g+Ehf0SHGVdhC/zKpXSeKnI537evDDSg+9QefPF9W/KbQB6d+vs8oRHHXxw+2O/0Gg+iaKXnm9Ju7ph9eE8t6g/aCoXe0NLWAbKC49vGPnqjdbrTXhSzoOjWromukNSsuI0UDKaOBNSnJsypSm6gWZQRLmbqDqeqyTKVRgAtf1VFCdgIbHF8EQkDwFuobkPx+eA8nZkBVzFS5OdBLuRpADWpdBx36CNpAo/fH4EZ6BN+Bn8DtTnEXSB9fvMAZmxTmTrYbxNcn3a1gIXWFkfF/lnAeKI1E9VKYv4Un4FQ4EyXjQjcLbcRaxlWGSKLfg8RUo7cVwPqgzlV7mgEYcZBS8B5+B0lU+Mgok2pf/d3A7dIIfQDp6BxRPupP4jyakX9XDr88dOpggbdlX/V+AqfANKJ1MeB+U9iiQDnV+JXeBjk0D5TsbFP9F+B7uBom2X4DCjgaVVwaBROe1C+wBiiO5ExRW9ZwEK6AzSM6CX0DXjvS1CPaDRBmKx2+g/ObCM+CA8loMOt9PwI9wGtSYVOB/VGNlMhmljgZupij6A4RFHqYiV4WlMqYeFdNAORs7dQZq5P2Oey/cari3BHUmGSDZFTbGXMUbDeqM1QmlxcOoYVdaiXIvHn7HrWPqnPrJkSAyFI6J+zVmq47jTGgAHhwJEt3VTgB1lpKdoSDmKjQaluBuEt+XcfNp3D2e7ZNxtzafwMDAvu9Ux7hvfKc723PibnXED8Xd17KdDHZ8fzDb+XG38nk27tbmXTg/sK9O+5rAvpzvgDpRyfagvBLlPDxUb1+UhureDR7zPdkeDzJ6JHfBdDkQdchL4RLtIDJyFsRchUbDNNwKI5Gx83jMVbLRoHr5IgPhJpARIwOvHUhUPl1D+2knIA1xK1yPuJ/OpYw9nQ8ZDfmg60/SEZaDf13Kr1qlnP+jaiuD/4eqtgxMwlXSwEfE/keVUkityB9TnEFwf2oVy5QmhTSwJ2XRXdwf8TJ9wXZY3H0RW3VOu4PuLNOhJFEaebAI3gB1aEorUW7Aoy/cDUq3EyS2i83i/m+ylawDdWJB+TC+s4KtOmF15u1BHU8kDhtrOqyVA3kbxoGOS/T/8GURDuWbKKPwUIf4PrwMoyFR9sFDx7z4gUlsx8AW8X2/rNpV57waFEeizrBBzLXp5zmcT4F0/zqMgkSRbv4P5oPCKD8ZVpJ/wdWwGxwILvii0QCJ/NT5fqUdRO7MmKvw5zU2fjzp7cHAseKcnwc8f8atOnWD32ABSFS+7Jhr8x9dAyvhm7h3LlvVScbFXNAIhYw3ifbXQ3uQYRF68S230Fe0jlZQfyg1jhl1tPyJxf4Ij/0TPc2+0UBAA7o7TmyX2uGnzkyNte5034JzoTRRx67/zmBYAo/DTZAoU/E4A2aDOj3951SGoKizUgcc/B8G3Qqbpx9Ed68q506gzu0cCIrfkcsvHaLg+xXIswy5jeMdYCLozvcTSCyv0ouAL9KnwugOWaJOMCgawXkmznVsR0BQRrGjej0N6jjV0SYaNAvxawuXgPKSEdEPDgX979NgAlwLQfH15vtJH2WJ9FaWroLHpQ+J/IKGiPwSz6P8dL4Tr0Ht+/pTmKCoPCUdC4YLhTtRMaGoVIgqsYa6fAtq/MIgP1EJNVjqBIwYDRSnAd0h9oDt4gcPZKsOaF/4AdSpTQGFUftVUht2BMfGgzqs/wMN3XeEoKix7w2XwnPwOyjdYIfLbmxk4AO2Z2kHaQNHyVGM7IPfz/B30CMC3d1K/DR74/Y7XD2emAbqpMojqus3oE5ZRtAQ2BXU8SkN1UeiOittddQSGU66C16tnQT5kP2dYXocpXk4BOU1dnaDZ0CGivJrDUHRKNAI0Lm6DCaB9H0YKP7d8AbI6PB1gbPcciwh/Xiq2zvljrkp4EycKrtfv/1xd910uMglo68x6NqTyH0cSK8S1atdzFU4MVMGw7egtNtAqCUt1LULR+V0oR4An4ajOrE/nuqzICT1MdVIrgZ+Irmb4TNQI98ZzgZ1erfBy6C70Q2wDNpAcfIBnjIW1MkuAhkD/SEoauzVEapz/wp2iW/bsk2Uc/DQ3b0MDBnzP0Nxnf27+N8Jr0JeHBkjbUCiPD+GJaCOR51heUX5jQKNLkg/MkhGgAxx6Up60530hdAHpLPFsDuosy9OFFZlPRwagvK4AYLyFDuqu9oijXKMhe8hKDounasjVh23AY0qbAtTQXGag+otP3WwFZGtCPwlrAMZTxXRG8FjonINh6dBbl1rS0F1Dor0eQaMh6+hPUhHMla7gMrwIvwCOgcKqzTk/gJsMGI0UGsakOU/odZyT37GF5HkyOQna1JMdQ1UcAJXC+rTE5oG6qUOsTuo05Fo2wiaQGuQbA3y86UjDqVTWifVluO641Rj3wz8tHAWiTqpxqCOVaIO9DhQnDYQFL+c28U91YEqrv7L6niUh+qRDr4oT9XDl8R931/bVqC7YG2Doji+buSvEQR1ZGnaiUtx6aoTVv1lxJQkKvMBIP2WJMpHxllnkF580TncA1rGPXZiq7prX+fZl+1xSHcSHd8h5iocpbgVt/TYKe7nbxRGYXV+FV+SmK6/r/L1B5VN5zECy0DGYnGi66gXbBE42AX3QmgAOofBa01laAPVKhX8H1VrWUziqamBHSnWb6lZtEqVSo3K7ErFNJHqtAbqeGP3Esp/Ho6CO+AHCHby7JYpvtFQZkATYDMN3M2ejIZkiB493AM6j0/Be1AR8Y2GisRJatja/h/JwjSS2hrQEJqGy0q7C0jtGmxeumx2dUcQtN43D2H2jAZSTwOnUKSPYACsgL1hLVREZhF4TEUimLAxDbzN77Qk6eIQ0lkKOo9fwNFQEVHcBysSwYQ1GqgNDYwm09NqI+NqyvMN0j2umtI2yaaoBmr7DilF1WKKZTRQIQ3U9v/IjDRU6HTVWmDd4eh5YlgkbPUJy3kx9TAaMBowGihVA8ZoKFU9KXMwbJ1s2OqTMheKKYjRgNGA0YDRgNGAjLuVEJZ5AJpAtgZKm9HOYSNh0kBtD6uGSZemLvVXA7X9PzIjDXXj2tM7wJ/BfnWjuGWWUhPI5kOvMkOaAEYDRgNGA0YDKaMBYzSkzKkosyBhG9IPW33KPIEmgNGA0YDRQF3XgDEa6s4ZDFsnG7b61J0ryZTUaMBowGjAaCD0GgjbPACt1fB76M+aqWCRBmr7WWxRQYzDaKAOa6C2/0dmpKHuXDxhmwewBNWvh93rzikwJTUaMBowGqjfGjBGQ906/2Eb0g9bferW1WRKazRgNGA0UEENGKOhggqr5eBh62TDVp9avjxM9kYDRgNGA0YDRgObNBC2eQD6At/cTdUzrjBroLafxYZZt6Zu9UcDtf0/MiMNdetaW0JxwzQPQB/w0edut6xbp8GU1mjAaMBooH5qwBgNde+8h2lIX4tWfQr7173TYEpsNGA0YDRQ/zRgjIa6d87DZDRI+2GrT927okyJjQaMBowGyqkBYzSUU1EpFCxsnWzY6pNCl4opitGA0YDRgNFAfdeADL0VEJZ5AI2piz5elQlGQqyB2p7AFWLVmqrVIw3U9v/IjDTUvYstbPMA1nEK9AbFnnXvVJgSGw1USAM2oTXx14jRQJ3VgDEa6uapC9uQftjqUzevqvpd6s5U/2Y4rxrV0J+028bTv5ztodWYl0naaKBaNGCMhmpRa7UnGrZONmz1qfYLwGSQdA3MJsVboDof++ktoU/iJX+Q7UFxt9kYDdQZDRijoc6cqs0K+jl73SEs8wBkNKhB1fCtEaMBaWAHGAoHQ3MIio6dBEeA34Y1xH0A7ArHQBr0hW1B0gt0l78TnAIVfRzWJh6vN9ugaE5Ob9Bjhww4FLaCROmNx7RET/YVbxjsk3BM9VIZT4V2Cce0Kx0MgUMgUT+t8BsEJ4DKVB7Zg0C7wI4g/ewFvkRwHAhd4h4qm3TdNb5vNvVIAzr5RuqeBsI2D+BXTsFqaF/3ToUpcTVoQAbxOfAVqI3KAl9a4PgbzAP9D/4BkgZwEQyERjAR9BXVq0Gi48/CcFC6utZug/KIOmd1wl9CS9Aogd92ylhpDZfAv0CGr4yVRDkWj9cSPGUoXwA50BH+D3yRMSCD4n04GnqDL+qsz4Ov4x4v+gfYyji6Cb6FlfBvUJnLEtVjFMhgkH52gztAIgPsdFA9JdpX+SpqeCmuEaMBo4Fa0sAD5Pv3Wsq7OrJVg35WdSRs0kwNDVRg1rc6vhGgjjQdtHx6UGQ49Ia94V7wxTcgmuNxZdzzRv8g2/EBt5xnQ6KhGgyvMLrLvkuOgByEWx25LzvieB4Utjjpgac640R5PcHjYvbVWfui+veGzuDXDafVE26FTpCon3vwC47YyWDwDSecpUqifs4gtPKQbAeXxVyFP4k6CRwyzurUQAX+R9VSDFmMRuqmBj6i2MU1RHWzNpsWeXqyrlbAlDtpGphBSn+A7mT7g4bYR4DkBNgLPgD5+48fcFqufuISdPt+S3xHfPsz261hfoJ/cFd34PvBzUFP3Ikd/rv4RRPC+LsaqQh2+r7/bN8R385huxMsAHX+KpfcMh6CohGGZSA99INM8MsnI+j/ICgzgzuluEvSj8r1CzSDxtAdtJKrEaMBo4E6pAE1lmpYg3cVdaj4fylqF3xKa7z/EsF41C0NVOAO6eyEml3IvobqJSNiv4U/aWyeDuz7owQaabg87u/7aVcdsDpViTq/R0EdblCC4X3/h3AEw+k/18Q/yHZHODOwH3S2ZeeioEfArccSemziy+04NIqyM5wOvuyGI2h0SD/B//157G8fD6xHEypPUJoGd3AfBMWVSY80fAOlEW7pJ1g+jTboMczdEMyfXSM1pYEK/I+qpUj60xmpmxr4lWL78wDm1c0qbFZq3c3omamQMWSk/mpAQ+rqmHQdqBPdAL+BZBbcAvLTsSicC1NBjzU0EjENNDowCjZCP5gEY+AoOB0cuANyQaKhd6Wnu2jdtXtwF+SDyqKwurvXyIN4CvS/OyKOOlR11irzf8GXM3D8098JbPX45FW4E5RuK3gDVoL+1x3gGpBxshZ6gkZevgQZRSqb8pJbdVgCkvtAhkMeqOwyGN4Dpe3LKThOgidB+vFlHI7D4VSQflS24HGNNmwDX4D0Y8RowGigjmngWcp7Vh0rc2nFfY2DauCNhFADlbhDUsdVklTmTlfGQFWkonnqpkyjJGVJSemW5O+nV5p+FKas+H46/rY8+rm9Eun66ZttEjRQif9REnLdlERZF92mkMaVihr4iEIdkIoFq2SZVJ/9KxnXRAufBtxSqlTRO91BpNUDTi4lzbIOVTTPAhIMjjqUlH5J6Zbk76dTmn4Upqz4fjraDgRfP4nGRgbHTgONxrSBHcCI0YDRQB3UQBfKPL8OlrukIvfmgAwHIyHUQC3fIamjawM7hVC1yaiSr5+di0lMRkSbAHo8Y6SWNFDL/6NaqrXJNlka0EjRn6B5AGEQTb5aAw3CUBlTh801YBq7zfVh9owGKqOB2v4fmccTlTlrqRNHw5OfQlgeUaynLnNAE76MGA0YDRgNGA2kmAaM0ZBiJ6QSxQnbPICwzdOoxCk1UYwGjAaMBlJTA8ZoSM3zUpFSha2TDVt9KnIuTVijAaMBowGjAaOBatVA2OYB6D3wZaDJV0ZCpIHafhYbIlWaqtRjDdT2/8iMNNT9iy9s8wB+45RogRstbmPEaMBowGjAaCCFNGCMhhQ6GVUoStiG9MNWnyqcWhPVaMBowGggdTRgjIbUORdVKUnYOtmw1acq59bENRowGjAaSBkNmOfGKXMqqlQQzQOYBVqvoSKrwFUp02qM3Jm0J8Lu1ZiHSbqGNcCzWM1V2aKGszXZGQ2ETQPL169fv2XYKmXqU/MaWECWHWs+22rJUcbscti6WlI3iRoNGA0YDRgNVEoDkUrFMpFSUQP6wp9GGb5KxcJVokwHE0d3pvMqEddEMRowGjAaMBqoBg0Yo6EalFpLSW5FvvrY0+Rayj/Z2WoN/Pagz/4uTnbiJj2jAaMBowGjAaOB+qwBzQP4NkQKeIa6LIJ1IaqTqYrRgNGA0UCd1oB5e6JOn77NCq9vNmiSWVjmAWj9CY02aPEqI0YDRgNGA0YDRgNGA0nWwCuk1z/JadZWcjJo9QXPMLwNUls6NPkaDRgNGA0kVQNmpCGp6qz1xMK0voG+4NkXNtS6Vk0BjAaMBowGjAaMBkKoAb1x8EnI6nVZyOpjqmM0YDRgNFBnNZDyizuZBWEqdm3l5+db6enpFYtkQhsNGA0YDRgN1BUNmMWdSjtTtf1Fr9LKZo4ZDRgNGA0YDRgN1KQGartPNHMaavJsm7yMBowGjAaMBowG6rAGjNFQh0+eKbrRgNGA0YDRgNFATWrAGA01qW2Tl9GA0YDRgNGA0UAd1oAxGurwyTNFNxowGjAaMBowGqhJDRijoSa1bfIyGjAaMBowGjAaqMMaMEZDHT55puhGA0YDRgNGA0YDNakBYzTUpLZNXkYDRgNGA0YDRgN1WAPGaKjDJ88U3WjAaMBowGjAaKAmNWCMhprUtsnLaMBowGjAaMBooA5rwBgNdfjkmaIbDSRRA1p7vFUS06vppNLIUJ+GN2I0UJwG9MmENpBR3EHjV34NhMlo6ES19RnlIDPZH1JOdajBvLmcYYPBXmRnTtCjGtyXk6bq1SuJaX9FWicVk97e+M0vxj/VvA6iQH3ihbqDrUh1URn/mcRCqpO8KEnpqWw9k5RWspNRHcsyCPRV1DHQMtmZB9K7HvdDgf2KOvVBuWD75LvvLyOhZhy/ooww23B8TRlh6tLhbSns+iQVOEI6n8OncE6S0qy3yYTJaPBP4mc4RoAawUYwGo6BsmQ8ASpzQeni3rmsxOvQ8bmU9ZQ6UN5LKGOY9F4Zle9LpOIMv4qmJYNbBsM7FY1YQ+HvJJ/MMvKS0fAo3FpGuNo+/AsF0F1vkKvKKFR3jteF/2QZ1ai1wzuS826wPfyn1koRkozDajTcwvm5CXQnWgDXgkRDUyPgbXgLroN06A+doSU8DbJMdZE9BdNgEhwNlRE17M+A0rkbgkPAyndy/NhItm3Al/NwvAlPgO4igrIHO2NB9VDdVAfJUFA6J4Pqtx9UVLYkgsrVBP4HwcZad3y9QXIivBJH4X05CIfu+FS2myAYn90iORXXq6A0gh2f7sQPg4nxY8Xp/QiO7QVnwZEgaQmPgPR8A+gcSpS/yvEuPAcdoDjZGs97QfGnwADwZXcc0oXOx42g60hSkv8uHHsSlNbt0BgSRWmonO/A86DrT7I9/APU+UkHqtf+8AIoPV1LylcG8RWg+twCko4wChTuTmgKkhPgFNComNJNlKvx0DnzpR0OXXeqb/AclpT+cYQbDA+B9Hw2dAHlJ3qCROFUjpHwBlwIvtyBQ/n6ov22oDo2hAdgO0gDXfNvw2joBr7oWuoPW/gege3/4T4CsuA1CF5zu7H/FEhv90Bz8GUYDl2nT4NuEHzR9SW96X+m/6L+k1UVnftHQeW4D/xyXI97J9D1KZH7vyAd6HzqWvBlKI6poPMZvNb7sK/rScfOAV+64lDdlOfd0AISpQket4Pq+gL0Al90PvV/UXzpzr/mrsH9N5BupO8DQOdG5/0x2BIkf4d+MAFUPpWzODkEz5dA8c8PBND/JnjuWgaOyany3A/poP/wziDdavRW53VHaAOPg+rwAPhlOxj3aaD/ko7pfOua1HU3GQ4CI6mmgQp80asTZffgoYQ6fMP+urif/nR5oIsoCxRef6AzYSVsBF0c6mgWwA8wAmZBPvgXE84i+QiXn36RZ9xxCFvF+w70Z9sAc6Ax7ANReAX0Z1Uab4FEZVLZpoMaB5VZ+/qzqmFdC1/Dv2A96IKX6IJXmr/AV6ALvCTR8ZOKObg3fvPj/u+zPTHubsB2GWwPQ0H12Bf2i7v7spXe/oDesBuoPjdAolyAxyw4EHqDdH0KSORWPOnnNJDOEhsClWEaSG87wB2g8zcYFE/lPxUk+oPLWGgPw+FnaA2J8jEeahjbgOqsa2EbaAW/wnmwK0yEf0JJ/iqr8jgXdof/wCSQqJyKK3kGxkAHGAJLQPmpEVSddd6VRhv4//bOAs6qouHDH2WQKgImLKECYncvYmG8ioUtYgd2oa+C3f3aQdjdIiaIqGCLAirKUoIothiAfM9z94wernd7gV12/r/fc8/MnHPmzPwn7917YQbsALb99TAM6sKZYD9YH7z3azgGHAt3wBBQ54B5HAfmk1ZtIt9BiySxCUfLcixY34fBflZc/r05b/76tg3Yl18A+9J/4UNQ9oWfwes6wdtgedUnsGkmVPhi3PvXAPu799SHW+FR6AD2X8u6MgQ9QCD0pZDmcSS8AZuDeelxS2gK0+AE0Dd9fxNqwV7wGViuXeF7CPPLDYRdLL2nG1iO1lCcXITMw34d2De5wbp9AaeA7XwpDAXVCyyHfXtJGA+nQRvoD7a17fMX3AVrgQuj/irrPAlsm7XB/nMCKMeK9bTsA+F2yNaFJPQDr9kPvoG6YNlHwTpgO70G54N6Cd6FjeF0sN63gWV7Eq4B9TKMgQ3A8tmHV4flYSYo89Df7cD7h8CpoD4Gy2HZLKOkVY+I4965Kx9WgN/hbjgabP+JcCbYllfAR2D9DgX73sGwGZiHfcjwUeC8sMBVhjVxgZetSjywDAbZ4HMhDOpQ/uEEHEy1oQnYIZtDT/B6B4R6DSZnQoUvTkTi9QPAax1w2TL/X7MTk/ggjna6ZZP4YRzNx2NdcKAtDVuAk8LnoCzL97CEEeSk4H3rwcVJeDWO6lr4A7zWgeh120NJKs2m4RAyeTzJyInluSTswDkB8hKc6DzXEH6GU6ENNIJcep/ErqkT5j0iiY/j2Dl1biJhfcrWQyT0SBIv4nhrEvbghKtPzWA2eH9eguV0ssiWE5Ntoo+284zkeDDHZyDIycxri0o/nHNOynkJ7Tg6SS0HlvMysM0tl8/JS3AB0lP7m/2pHig9Df1uGcLm/yWoneDVTKhwQ/BIEvZgXayDdT8HHoBcyiPRTUOQi8LgEOHYAjaF46Co/Htz7n4IepdA9yTiGPohCZ/F8c4k7GEHCO3+CWGfE2R8wyTyI8cVoAHMgo0gL+FhjqdDUF8CbnKyNZKEPVKJHxDeCg6DdPvWJu5isAE8Cz0g6HICzi/2Edt0c8hLuI+jPhenLTnpfbZZwHZXe4GLVV5Ca462SwfYAhyvamd4MxMqfLFP5IP9aw4sCWpFcCwqy3YF5CXszXEMqAlgvTqC99pvsnUlCZa3CywOjUCtBHkGkOHbIbTvS4T1Vtnf50KYBw8m/Ciol2G/TKjwxXI6dh1nMwuTMvPvNYTzEmzHMFd+Qfgq6ASWP4wbgn9rFUJfJbGmHC2LR6UXr2VC/7yYt3U9FCxf0CACvZKIPpiPxwWqMqyJ86VcuTrIfHnQQszUzjcD3DhsCNdCSwiD0Ekil+zwx8Jv4OBVRV1bePbfr21ImgjfJqfCM/OIO9gdHNvBaGgIYZA0I2wnd4JRYYAYXtkXNLbw8Pfrin+HKu+LmU7IThiW9SDoB8oJ4kDYzUgiB+8v8B84Bc6DL8GF8BVIyzZxsgoqINAiRDgGv036E0rju20c9AcBJ3Y9mQPXQ1rmma32JPQHx8Q7UCvBxWoyBE0lIJ0hV3pX0m334BXBzCTf2EAi8/wLnAjTCu1tf5mVnJjN0b64O0wDPbVs2cr21PssX/A1TJrZ9zUhwXYLsmyTQoTj1wk7cUy3WXb+36Tu8VxoQ/1Pl3d86jrLZLlzKX1POO/CaPqlISE5hnFj9GdomaRnH0KZTLcPmJfPL4Ag28Uxq296MQWCLLv9xPG5GFwAab2djhQRtp92znFuJdJ8ZrrffEjchTAty5Tud9ZpCOiNPvwGyjFQNxMqHK9rEV4/iXuwH6ld4Ux4E8yrNzwAaZ1H5Fy4FSzjADgRXKBvgHVhHMwF54Eg+7GyP6jvCw+ZMan3QQUhwFG/107FDerNBpBOD8/pRrplfh3M/79wLxQn2z7MF8sTnpB1cQFx66mK69eeT9fD+CKv0KkW1YruSMVaw8Ckgrdx/Anc8TrICsBJQtnhQwdwcDlILgI74alwBXhNWTSei7cAn+cAWg/UOHBh3RkcCE4Ob0FTUE6mG0Ij+BlWhyDPqQ4wFjYGB4HPCvo9BCp4/IX7n4IjwEG7JygXzevgPiNoVXCQLQX1wXotASfDleCkkpb32y6jk0TDThYVUa62mUaGbjicWH5IMvcdRJhwkqTMpOSEaFuMAe8Jk4qL5uYQZFkPgQLIlW7drNe2oOrAXjDBSKJQLtPtF6ozeI2LRLouhxJfH1aBX8Hy3wnZ8rlbpxId2ytD8DWdZ+qyTD1ttyDrG8puWks4EorL375RVP6cmkcrpWKtCQdfHIeLJedqcXTMBIW8pycJ+3P8KglvydGyBVmX4GlIC8eQT4h79F7HUJBt3wr0bWIS5pBR2Jib/2w4BApA2RfSC0wmsQwvlmMSdE7d053wp7BOKs32sV2DmhFw0bw8JOQ4mvfzcHFyzo2HmwjHaFvYB+wv+8HN8ACkZf+7Gk6D9vAiPAlHgmNpF9DbCyF4RPDvudVwcUr3iTZcWJB1seV/Dc5L0pfj6Fxqf1kV9oU6sDc4x98LxSndD8y7ddbFxm3/dpC+Nuuymhl1gCxq6kGFxsMEeBZs/P+CciedBy7Yd4MdojEoFxUXvofhD3AS2x7OBq9XLuK55ETvM9NsQdyNhh17BNwP18MoeBAsizoWbgIn3pD/HYQbggPFa130gvoRmAmPwiXgAL4KLG+Q9VLngOGORnIon7SDU2yY4xqfZz7BFy9xYrkAtoGN4BFwEqoFXn8AOBE4gXwG2bqVBL3ZCrYGJzPTyqJfubgLdCjmJhdnNz2WaU1wUnkInCzTmkVE/1aGVeA6aABLgpOjE9TR0AlsQ+tZVLo++Sz7jGW7Mgn/yTFoBgE9s1z6tgd4X33I1m8kWJaW4OR9PlgupQdtYEfw/s2hF/j8W8C+NgaK01ec9Bnmr56BNeBYCPWtTbi8+XPrPLJvdIdNwf57MyjLeSTo9WWQ9sJ67gf14B64C9aBXeFRaAhBqxP4IERKcXyca3zmSaBvN8J4MA/L5vjvDLvA4aB+g/5wB7gh3hkegyag+kC+gTJI35uB99pvbGdxLrL+jqfd4AVYHk4F2+dayO7PJM0jx9aJ0A2sox46duyTV4B1t/1bwaeQLfuX/cnnrQCLw5egD8uA9+lBTwh9k2Cp1ZcrN4XuCQM4pnUbkeNgT7BvOndvB45b5w69sAx58BmURc9xsXU/C/TmSrBewyEqhwN1cqRVqaR69er1nTVr1nmlKJQTSlMYAwVg5+kPx8DXoF6D5rAa3ATj4Gcw/Qvw3FywU34IraEh2KFqwefgPWk5gCdBQRYvEx8Bz4MDywXpQTgCfoF3QdlRR4ODUtlZR4KDsg1MhvNhDgwCnz8YnERWBSe9XvAH+Bzr8zT8CUuB6c+B6WmtQMRJKi/FD4Q/BSdsy68mwvJwM3wD6gP4CqzLptAfboPfYRgcAPuBfp0JliGtd4j4rCPBSfdyeAjUijAUfjKCsuOFqYUT+wZEnFCnwFSw7dXSYJt/AnqRBz6rBZwAH0Fa5mF7Hw47wKPwOuhZyGMfwruCvlwG3mPeudKfIN1r9wXLdgzMBMvlRibkaRtaLtviZHgPFoMm8AIoy7oUeJ3t3TuJD+doH7FOHcBF/SnYC5xcPwYn2j/B+20707Jlf28P9eB9sA3NZ2/YDWyLS0Avist/BudHgbI+I8FnOsc0h2dhC/gWrHdXuAEeAKXfm8LuMAS8P/STrwhvApa/HywPR4ALxWngtWoxsG1OAeuRlveMAJ+vQtyxa3v5XOv8Kdhef8A40OOjoS3cBLan9RwEy4LlcJyeCW+AyocJUABpLU6kPgxOJyZh28l+tx0cDN+Bbf4j6GNjWAceg8fBtrGt9agPKPvCM5lQ4adl1vFpGA/vQk8wf59/EfwFz4N195m/QC/4GdJyTK9GGpp5AAA4t0lEQVQCh0MnOAPsq9Z3AzDdvPqCbf8SWJYPwP5eG2zzJ2EuNAT9td18rn3DMujjcTAG7DfLgeXXy7ehB+wA5n8+hPLrRQ/4DY6F7PLXI20ZsK7mG3whmBkf+r4j7AeTwPr8Co1gJuixsk72Qedk1QqeAsuxwFSGNXGBlalKPWhhf+mjSpkRCxMdmD8OuGkYNn+ynifXs4hdO09K5UZc9K+v3CxjbvPZATfhu8znZyxS2S/sNdEdYFR0IDpQsx0YS/V9J+e70Ooq30EeAedV1wrEckcHogOV4MDC3lVVQhViFtGB6uBA+Ai3OpQ1Vxnrktg014mYFh1YlBxY2Gti/KRhUepNsS7RgfI7MItb/Tt6ddVsCj6juhY+ljs6UF0ciJuG6tJSsZzRgehAdCA6EB1YyA4sapuGvJSf1xFunopX1eAVFGzlChTOj2T9Vu/8kN949mNfdSm0zIQW7Ivfxr6ylI/029Z+Q11dAG0zoer34hf69qikYts/7gC/ub50OfP0VwINynlvRW7bhJstt32vvHKOC/12G8I9y5tRNbuvN+Vdu5LKvB75+IuN+fkl1lDUWgT8VYLy1zb+cqWiWpIMboHnwF97lEdLcFOb8ty4qN2zKG0a/CmTP4cJ6k7An8xUZVm+XvBtOQvpl78+gTnlvL+k28ZwQViE/UmUC9CC1lI8cK9SPnQU14U235Vwddg05qpaDxIXy3WiHGn+bC8ProEfoTzyJ2duHBa0LPMHcF8FHnwh9/pTOtUBNsuEFv2XrlTRTX9l6HIy+RwGVEZmJeRxNucPTa5ZheNWJVxfmtN7ctG64Bu0CaW5Icc1g0gLm5kcp2tO0qK0aXAH6Y4yH1xMlV/u2gn8DW5Y/Ahm5OSxL5T0Ln8ZrtkDnHCawFqgOoLv3PKhE6hVwc3KluCOWXl0x+zvrM3HgRC0HoHR4C7WhXE1KEp5nPAd6ObJBS4q+4N1bZ2k2Z6eNz09YTQl3h58x/UfUG6y9GZn0KdsOcgagmUPC5jP2h52AcusvMZ3NKvDdhC81wMn6zwIakMglNU043kGEln/fWBFWBP0O8h2sP4bh4Sso+2ix5uC/UDpx7awK4Q0ghnp/QFgGYqT/cNJJx/ML8h6269sU/0Nsg/os3XxPn3W366gJ0GebwH6q2/pvNcn/jaoxWEH6JaEOWTUjlfv3wjs+7lkOdaAyfAd/AW2peXeHfQrLb3Q/90gbL46Ebaf63sD0Dc3ckHGbSfLaTnagv0q9Cnb5UCwvGl5ne3pOKyVPpGE7X9+ymTZv03S7NOO2XwIfvncDcGy7wzWL6gZAX32nAT5bMdIOs1z9jvz7wy1Ici2tk76ku2Z/hwEtntRKu5+y34w5EN45vKELZvlDOOBYGYBTXvrdV7TEZw/jBclr/E5+pFWHhHvtW+G5xP8W7afvod2MK6v9h/9Uj7XcnaBOqA82mfsR15rP1V54LgI9xL8W44jy+ninO4vecRtr1UgrWWJWPZtIDw3fd6w857tNgomwXSw/I4n2zlb1s9n2f+CHKPOK9bFvpo9BsLYcE70nPXwmcYtl8+xztl9h6SoSnegDN8UvZSHOyn2AxtrGnyYxIdzFGUjPgVD4UooADt1LtlZvoCrYRg8CbeBehsegMGwL1wI78P58BrcB6o9fAuPw6Vgp90W1GlgGZ+Fy+E7cNBka3sSzLsPWI6boSW8A2PhDLBDW8eH4SKYAvmgToDn4C24BRwQ1usGuAtMrwdpnU1kLtwNS8Nn8An0hyHwAeilg+RLGAPm48AYBG/C9WA5DgdlHfUpyHjfJHI8xwLwHj2ZDPqvHz+Bfuu95U7nQTQjvZwN90Nz+Ags0wB4GSx7qKN9ZCRcAZ9DKB/BeeRE4X36/hKENl2LsHXWvxvhK2gDyv7wItyThF/gaL+xr02E7qBegdfBdI8PgmoNMzKhwr6j7zeD7aYvDUA9Bba15ToWcmlnEkeBfecUaAHWR4/McwLorzoZ7EuXwLMwGWxL2+V38Fkrw3uQD0HGbadWYFt/DPbLZvA/8PwV8CnYD9U+YL3Oh1fhCcjWuST8CNZza9C36XA9DAHHQX1wQZsK1tPnWscgF+TRYPpe0Auss/feCd/DtqCc1EP+lmk4NICmUAA+9ybwWdZVXQWOA+s3Bk6DbC1DQgF4/40wDfKgNjwP9s1L4CMIfcD+aFtbjnvhO7A/DYQ3YBCoI0Bfh4Bl+BpsC/Ua2P7qMjB/r9GP3qB2gXFwAdhv7UvZOp2Eb+E56ApeY3/Sw2NgV9C3G8BzI6BRguX2uttA3/qD/d566H1LSGttIpZTT124e8JEsC1uB/P7DyjraR/VV/vPm2B/yNZJJPwMT4J+dIPxcCk8AwNA1YL7YQicB5bjfKgDeqO39mc1A/IMoCXA8dEEtgbLYXkfh9bwPgyEa8C5ZjmokMqwJlboOdX25jIY1JhKzklV1MG5dxK3YX+FFcEJayTYSdRaYIcOcdOU99gp1zOC7JDfw+GwOPwBB4Hy3K3J0Xge2LHUgeDADB36BMIPg3oIHoPw7KGEd4Rs2VkvShKX5nhLEnYgnZKEHTyhU5vUE8Lk4oTzLNQFn+UE54QR5MLmhJCW182FBkmik/xhSdh8voEOoD+zoQWofWEk1DaC2sMv0AQuhwshyHhf8Bk/QZhEliXsQHdi6AizYHlQW8GnmdC/X34jyXuVg/64TKiwLFMIrwMObNujHqiV4EeobyRLljV4an+4E6y7k/UOEKS39is9s92PBeUk5YQbvOlHuBfojc/sBqoR/A7NwT77PCgnoNDHjD8I+xlA9tnTM6HiX4LHXnUd2E+CjiLweBK5kmPbcILjl7BZErcOrZLwexzzk7AH47aT5+fCaqA2hkmgb6oZWOelYSCcDcqxZB/Wk2zZzuuC56aDeQbZZ08C28/n2k9yyfqemZzQ+/fBdlIXwE1g/tMg1JdgZuycytF2to6LgbJ/69M64D1hfCxD+AewnmltR8RF0HqqfaAdrAA3QChLe8Ler5xjHKPh3JuELwFl/nPAMtsPP4E6oMx7aCb0z6ahE3G9a5ikL8Xxe3A83QwXgzIP2yHU07SgdwlsmURe4nhZOMFxAnRJxZ8gbNvap20X208dD19AaOcXCetlti4l4fwksSfHsRDqZzv2B/UR7J4JFb44NnxGtpYk4XewPM4NzlurgjLfSbAy5IFjIJRvG8IjQNlvBmRChW33dRL2sDHYVuoMsLzOdephCP3cuPmXZsx6bZEqw5pYZB4VOeEEuCjLwarsNA4UB85GsCK8AkEtCJg2OSRw3A4mggNGzYQZMBLWAs8NBDULfNaN0BTshA4QtSHcCt6v7LT1M6HCj3MPIOzgqgVrwCjI1h0kPAlbw/lwFKj14e5MqHBn7rOCXCSdyJTpLj6zwU7upLUHhEGXR9jJvCQFP83HeujnX6AXYSCtS/jlJJ1DZhA5aXU0UoQ85/3mo76FDzOhwhfPTU3iHn1uaRTKaxnNI7S/g9pFJ2gJAk7aLg5p6e2zYPqFcCioj2B/OBDsO5tBX2gLtcH2VmvDIxC8MX4XrAb2pcdB/Qz20SXBtnobvMa+Og46g9Kn16AlLA7XQllkftb91eSmxhxdhNRFcADYt9cE61UXyqKfuPjT5AbrUR8GJXEPLkid4E54FHrAc9APbKOiZH1to7dSFwwmvB44Mf8Ko6E0sl853tRXYN6O/UYwHILMX79ugd/A/m6fsdzj4XCwDZ6BoHoE9O+lkMDRPH+B6fAihPvnEL4J+kCHhLTf9rFQzu8IfwLqR7CPhWuHEDYv5bNuz4T+ebEO+v70P0mZDbPl7A9PwT5gO/WDP6Ek2T9Vc3Dz86qRRPrm3BkUxrF1GAuhna2HfpWkUVwQ6md7OefZr+xHp0EvUK3AMZStdUj4Ahxj3ZOTvZOjB9twaSgA2/hOWA5aw3ugNoARmVBheGQS9rAhhLjXXQXWzXG2G9hW+qo6wiuZUDV+CR2vGleh2KKHzpa+yAXPwds3nUh4WlbcwTA5lWanbAEfw1GQnhicvJeFi2AinAEuUMqO5POCNiLgQGiWEDrjKsT/gEmQ1lJEzNPz+4M73mPhGXCysWP7LK8bD0E+x4nUAeGzQse2HsOhLyg3K41gnJESlMtPb0lPNLOIO8Gn5cD8LUlwEAV53UxwsDcIickxHS/quVm3/Cua6z7bX8+Py7p6ala8MfEZ0B72guthGdDHR+FI+BDUZ2B4D9Bbn6Fs+0cyocINQWivPUn7NEn34DMsq/3Ne64E28k8z4Eg29h22h6GQtp3oiXKct0MLhRBPrcuWK+XwLqdDsOgKGW3YbjOtg/yWR/AoSEhOU7jaHuvCFvALvAGrAnWLZfMtx743LDo2KfCIpF+LsnFyvpmy/vrJITzIf9fSN8MbLud4BJwQZkOH8MhkNbX6QjhX2FzCPdfRLgNOH6HgPGrwEXtbQjSv7RCudJphi130JIEwjgLaeYzBrLLafkdeyuB5bNu9qlN4CMoTqHfebRNbBvnLqVvoQxzCafLnQ57bWmU6x77gHmfANYjSK+z5XgKvjqmHgPbUFnWy+FzeBY+gStgAvSHEaDM43+ZUOGmZWQS9rAtDE7iXndaEl6R4ww4JYl7yJ6jU6eqTzA9+KtPqXOX1MFRCxbLffrvVHd6W4ODtABWgRfBTpiWE4IDyIWiAdwI74Od2M7xFgStQ+BucLB5bQ94FxxM7ujXAGWn7Q63w/rggm+51YYQOncmIXk5iOO94ADtBw+DE+zaMB5+gV/hGzAPlQfHwM3gc96BMNk6QFaFb6EAXOiuBieQtPTDui6RTixF+FWu6QZNk2t35Oii9An8AKuBqg9dMqHCc9ZBb5QD0TqWVXpZUnlt/03BuhXA0vAmWMa0rMOT4HX3w0CwLS3Xh/AMTIKzYRTYPtltaD8JbbouYb23nqbbBouDOgduyYQKP871nnGQB6Gf6tUdEO4PExpJmU1jvoESZN1tDzcnBWDfuhSaQVvoDYNgBXBc1AGV9vV74qtlUgsXQq/LpSEk2vfMowBc0Cyz7fMQ7AcvwckwFVpAUZrCCSfyA5ILGnG0r9jXSlJp+vA0MvkS9EM1hJC//eAFGANu5u6HlWAoOAatTwHUg5HgvWntSsR6hvvvI+z968FoME/v2wGy+yBJJaorV7gYqYPh+Uzon5fXCHaC+lAAzvk+Tw/7wVGgj6eBHiwHpZXj2bFwSHKDz9gXStMuyS3/OswmRU+Lk5vFN2A7KICJcAfsBtlyrIUxOI7w6uD19qcz4EjwmevAbWCbtIOdwTlcn4xbT+W4nZQJFV5ju5m/vi0O40F5jX3BNi2A9vAE/AHVWuXppFW1wi5678NUcIIuSoM4MRDsHGMhDw4DJ5e0XifidW4EHEx2VAeb2hAuy4QKX27iYIczv+/ADmnncIGx8zixDofl4Wj4AvaH0JkJ/mvBMU3dDbuAnfYnsGM6OOykK8OD0B0OByekAlgJnIwt71nJkUNG7/H6AHwMDvpvwYGeS8NIdKHbPNfJItJeJP1e0F8H0LKwF8yCe0CvP4Da4GKr/oK9YQDcAG/BFMhuE5KKlW1mnl2KucqJ4ALQh0+gHRwPv0JajxDZA/RJj5pAN7AN/gu2p3X6DMJE4AR1CSjbwEn0UyPIc6G9DZu/cX3Qj/OgI7goTwN1LXjua7D/+fy54P0XQZATmRN1HdDLonQpJ2yDcTAd7EvmORXuBr2ZCD7rNTDfV2AY2Jd2Ass0EA6C2WB6Ltn+Z8Kb4ILZFk6FH+ByeAIOhWYwBN6A4nQAJ+3rx4HjSP/ugxWhOJmvda4Pk4q50PwfgmOgBVi+u0GPjoKxMAPsB45H+/ZJMBRs4zZgfb+BtJ4n4pgP9y9F2Pt/hlNgBOij/fFXaAVl0S9c7Hj5DeyHu0FaltP+bTt+Dq3hv2CfuhqegX1gGbAML0NZdDAXPwqHgW35HNwJDaA8si7mtyQ4nxelnpx4DPYE2/YDsL2y5Vi5Pkm0vzim7Zt14HU4HdR1MBi+TJjG8U+oCz+C7bQu2J4XwxHwLOj/h7AtjIQg2/dkcBwVQENw/PwOUfPTgXJ86cPGKY28rrgB2oXzq6QysgN3TcWzg41IcLJJy8nm3iTBya12+mQZww7qFbLusQ4OriAnuDyoFRKKOTblnHmWpNL6mZ2P9+VlJxK3bKYvBkEO4CPBo3KgfgttjZRRpS2vvuVBSW3iJL8SZCuPhHrZiaWIW28n+KWhMbgIFSevW7a4C5Jz/UpxTbjEtl8uRFJH05un4ulg2lcn6Zbpk8WEl+BcHoS2DZeGfmD9yiLHrGOtLLK8Pq80sl62S7b0xU16tsKYs88WJxfUXPfbt8pan/AcF657wL5c0ubJfpcHucqpp7Z9RaRvJfXl0uZvfbL7S1H3Wm/HaFlk30/353Cv82EuH2zjdN0cj7n6SMgnfcy1LqTPlzlcjjWxzM+o1jcsRIMuxrg3oAfcAa9DSQsMl8yju4idNE9KjBTlgO8w9PlgeAQGwaKo9amUn05Uppzs9qnMDGNe1cKBsGmoFoWNhawcBxbimlg5FZjfuSxEg9zp+lFfH+gO7tTLqq7ckOtdalnzqQnX+3HmoaDfB0J53sVzW5VXa0q4TZUvZSxgdXBgVQq5RXUoaCxj5TmwENfEyqvE/MwpGjQ/3Y15RweiA9GB6EB1cmBhr4ll/bi9OnkbyxodiA5EB6ID0YHoQCU6UNM2DaX9MlQlWlyurPwCZq4vTZUrs3hTdKCMDvjFr82LuMc/t61WxLmYXDEH1uR2vzBZ1VVVy6l3/kLBP9tE1VQHKvGjmH542Kqa+Pg/yhl+CrQwivwgD/Ub55WlB8iovD/BqqwyxHxK74DfhP+xiMtPJN0v+JZX13JjuzLc7M8rby/D9QvrUn9dUR5fLuO+jkmh/fLvHkm4Kh+ep3B7V2IB88nr1Arm53fQCuA52B0WWVXimlguj2rKJw1OUjvChHK5VLNusk9U5oTgpzt+kbS8coI6E8KnRP4kyi9LHgbqIDDu4lIa+S5pQUo/e8OLOR7qLx4suxwLJf1sjkv+lj6U5fq/b6xgwEW/ZwXy2It7yzLvtOH6DSrwvAV1aysetHE5HrYn99Qtx32L0i1+mbO047eoejselgM/aXisqItieg1woIy7KifRwfBugp2xE0yF3yH8mwn++wlvw6dwP4TfSe9P+CJ4GoaCvxe+DsbCx3AIlKTNuOB1mAKfw+mg1gGfNQC+AZ8fJkPf1fsuZRJ47/MQ7iP4t+oRugK+BMt+Gih/2eFk7v0FcDG481bWw3eH42AyhB29v5/vD/78bwycD+pRmAsF0ByGwYUwFXaGN2EVCDLuxK52A/89C5/lwPWngP58MuTXgvBh8An43CfBgV6czs5xMjvNRbe0Ksu1pc2zNNfleq7tflJyczOOtsF6Sbykw0ZcsENJF5Vw3n/34wmYBvZvN2CqJfwMN8IEsI1DuXoSvhzU0nAveI1tegAE5RN4C+wLjknz/B/MBvvhWuA7wlFg/3sR0v2KaEZTeP0DXi+MZn554tixzK/CGkl6+mC53gDL79gdCRuDcqxcA44f83gFWoP6D3wIludlaA+qqPTCs4Wv4zn8CSOSxM5J2GcMhbWT9PThKiKzwDpuCIPgNrAMzhFXQy1Q+q8HE8AxqX+51I3Ed+ErGA37Q9DxBKybbaI3dUEVlX4k57y+APpBQ1DOT3tnQoX1sn6Wazism6TbtrfCB2A7OR9dCuanJ9ZBfzeB7+FH8FMXdTJ8DvoyEJYCZX6XgOmnQ9DiBCbCHLAdVgXztQ/4LI+h/W3L2+E9sAzVTmVcE6td/Spc4DIa5EA4IXloV44uTOpaODcTKlz43yHcHGqDnfJKUA5YJ7/NYHl4AJzo6sGy4MBoA0XJQWgn3Ta5oCNHJ0nvtRP/BXZadQG8lAkVTsIu1j6nA3wL6UFBNKMzeX0BGsDSYP26wKXwHDiom8Br8F9QX8AAWAycYP8A6+Zk8izogYPSyc7BZnwuuKApB+E9sBosA5NhdQgy7rl24ES3Jqjr4E6oBeZnmc3TxUjvlZOmE0RxOjvHyey0PqlrzPsiuAL0JV3WHsRfBa+XQyGtw4l4jxyWOmEdbI+joT1cBj6jEaiV4WKwH5muv9lKlzGc04+TQoSjzwl9sQ5h29D7zoMdIagTgdvhHgh1WS6c5Lhzkm6fPwvqQS4NI9Hy+lzbcAp0hpYwF04EtSdMBPMx7S5Qj4Pltc/owZfg5Gw/sS/kg7LfPp8JFT7Dfqamg/1dnQH6mq3NSPggSVyJowvMVkn8QI5fgT6m5Xiz/KcliXpnP10MesGLsATUhXvhWlBes1YmVNguVyfhotKT05nDBryOThJsi+/Asan2gWkQ+otpQV8QCGNmEOGh4Ni2H3uP9W8MU2EHUB5tjyWMpNSC8A8QPLUfWA61E3wKK4A+2B72vaLS9Sxcb/veCPY55b17QwOwz+wCyvoatz1sm5mQD5bnEBgOnjO/2+BOUOfAVZlQYb5uLGxr28dnPgbK/vYW2Ffto2npuXOLWgq+hVAufZgB9ot94TfYGjpCtVMZ18RqV78KF7iMBjlJOAj3BweGnU7ZWbtmQoW77E2SsIf/gANVvQ97ZUKF73p+J+wAzU8YwrEbFKfFOemg3xSOAQdOa/CZli3IyeDDJOLgNB50C4HTQyR1dMCEepjcFKyj928OQTsSCHn7zI3DCY4FsAbkgxPMxbAluEgpB7QTroNbjYctMqHCFyfQ1bPiDuLj4e5UuhNaE6gF5ucEUw8mwYOwDzSGknR2jgss88EpwoTjpTeBz1LWyXPp5/TxRA71JC0/ld6F8AGpuEHL7SJsvo3A+sitEPxqmMS9Jq1cz/Wek9IXEc5VXy+5MOu6jYjbN0tSRy7YL8dFzUj7A+yvQRcR0D8n5Blg2wXZj3zmiXAXWPbZ4IYiP+EhjpeBaa9AkB65kVBTYNVMqPDd3nOEDwLLk0uOiw+SE4dzfCLroo+Jb5+V5uLgApJug0+IOybVkqAvu8LT0B+U42sw9IDmEFRUejjvcQMYnSTYN61XWu8R2SWdkIT1dc0kPIij9wYZ3wt2g3GQn6IgCXOYR7anZd8aesMcULfBGZlQ4YtjQh+KSh/IuX6Qn2AbfQfqedgbdoICyE9hObeDA+FVSMuydQB9eBgeAXUOXJUJFX4ae3wS9rAczALnEzcNvSCXvC5sGroRfj3roiHE94d9YRhUW5VxTaz0errgLEq6gsp8CQ6Wc6ELTIO14W1YGlaCERC0MoFvwAHUDpxElBPkF8nRuHIQjMqEcr84yd4MncHnDQcn1jD5/ko46C8CId3F9YdwgqO75FxaisT0dTOSi7w/DGiTTHfxCpoZAhzngM8dArvAgXA3qK4wNhOa9+X7eaPzxGonMb1Nl+134hLq6GUO/k3gMDgRnJROgZugLPqDi4emblglCbfm2ApOTZ3Ts60gtGvq1DzBfGL2Ba8NMq97QoSj/cENi/q58JDZkD1LOHj8C+EXYW14F8qquckNjs0ToDn8BC54em2/KUlbcsFu8CM4UU+EbOmLZdXLIPuNHijrF8pi/AdoZCCRfc7yrAGhTJ8Qfg+cwNN9xnZP90+iGW3L6+FwKNwKl0MfKEo+Mzsf4+m+Hu61vHNChKMeNobNwDadAO+D1wXZ/y3PweA4vhZ6Q1HpnMqpspQzOwPbJMi5Q4/DhmurcIJjP/g6FTfoRs6FuC240fkYghyf6TbRD1VcuufTz7yOeHo8W665Wdc4l3wFLSDdVusRfxCmwnvg8/UpW0uRkL7PPulYcNOg0nUoTPn3a1H+N+BS+3U6/3/fHVOKdcDGWFTkxGOnuB4eTejB8WlwcLkQN4Y6YAd0kjfcA26AdWAUuNApJ1PvOc8IWg/2BHfSRWldTuwOy8NvsAxcBQ784jSGk2uDk67yXZCDPlsuWp3gzeTEZRxHwufgs0eDMmxacXLCXhKOSC5yQHeHvuBEUJRcABomJ/Vz2SSsL/lJ2IOL3FFwEIT8ViB8IPRN2IPjhXATlEVOpgWpGyyTckIZDmFhN620cqK7AMICmOs++0S2nORXzUp04jO9rHJCbpTc1Iujm5GxSbykPpRclln03TScnCQ4eXcLJ1PHSYRt/9YwPkm3j4fn2YfDgmKZ2oH91P6npsNPYBntg+oQ+BGs++pgfWz7tmAbb5/EOWTqaRnPhkthYxgMfSCt0HdM+wwOgJCv49ixYnq2liPBvukYtp6rgePDctwKPlO5MVgG6sPpYHkuh3XBvmSfyJXem/S0sst5JCdDORcjrG+5ypm+L51fOvw5Eetgmf8A+8J5oNdp7ULETV8HsB/bnpZBhbmjMFbYVo7HotJ9pu3uc1R72BXS5bU++qZfv4HP6gs/QLZOI+FB0F/luLd/KfMM5TRPvb8HlHWYDqGu6ed7Ppcs+5pQF5wr6sBacAPYL6Iq4ICmLipy0XoI7LwzwU5yPbQD67k7PAaPwgB4GPaFqXAP9IKREDSYwNXgpOHEcyIcBWoTOAX2NJLSJMJ2/mNgMhwGP0ETKE7nc9IyNIVW0BnegmxdQcL90BCWAMvvpPcrWCcXiLpwEnSH4uQ9A8HnzQLr5KByUH4PPutcyNbrJFwGPs9neK16HJwQboZ3QH/C4v0dYfO7EA6BZvAhWP7noSxy8nSyMY9vwIVhKagPTlZzoAu8DE6uu8IYGAvqT1gebHcnzZag1/pv+WyL36ENbAz3gcoDn+NRjyaA+hgOBes8GtaC1eAOKEnWYxnIAxeWQ+BeULavvqnNoRvcCV+Bsvym65/3ugl8DrwvtIn9qTfoUz2wnYOso+34NNjuHWAzOBkWB699EKx/T7BcUyBIn8+D+8G2bQ7HwqYwHnym9w6Go8Fxp2bAOWDfyod+MAR2AK/Nlte3g7PA59jHLNcgOBBegVGQrdokPAID4ABw7E8E+8KO8CnYVruAbeicsQn0h5dhG7A8RaVzah5ZzjywXheBffFReAr2hzfhPciW9/UG7ytKr3PCMj4J+q1XLaEPpGWdVoYe8BfYL+3vTeAWsAz2jelwJuyRhItLv4xrPoeTYCCkNYLI22Ad7R96tgpcANnS953AcdIR9obJoOznwSP74nD4DTx/GljPuVBavcGFn4Hlsq/sCV/CUOgOURVwoFYF7l0gt/r3m5kzZ5a2nKtTKCeBOvAcvA9LwhHghPE41IX9oDV8CE/DHNgIfoSxELQcgYPADuuk8wUoB+Zt0NVIlixDN/gDHoY2YOd3sG4KT4Jy0UvH1yK+G0wCJ0EHjRNFttYgwTqa/z3wNag1YWdwsnCyHAfKieEV8PkqHd+Q+PbgPcEvgpnFcmuOt8OW8DI4CSr9PASagn62gmHwEzQBJ2jPvQhORmpjML87jCCvaQRjwLL6/KJ0NiechIP0d32wrZ4A6+zzPoAPQZm2AcwG6+4kFNSAQCj/FMI+P9Qtj7B9Y3EYDw/BTFgBtoW0nJCCp/apg8C2/hxsF/tUWn2InJdOIKz39jFlWZ+HGUZQfbCcS8FHMBE6wb0Q5P164bO8Vw+U6WuDZX8ULPtrYJ2ytR0Jm4ATt3l71CMXJvutRxcjvbCdToI1oCeoLaEz/Ar3wxRQS8CBoHdO4vYHZR12BZ+lf/aFZWECmDYLsrUXCSvD1WD/8x7zHQ3Wz3KlZX6OI8u2I3wGD4LX1QPLZX6jwL6bD47VxmDejk3vtx3/hKLSOTWPdifWGq4Bn2NeK4H+mf8cyFYHEvYAvfNe+49eqC7wJdhu5ueC1xYs233wO2RrUxK2Bb31ms1gBEwDy7Iv2F8fh7GgikpfjnP7QSOw/7wKyvwtZwFYrr2hHUyBe8G5y3K2hHCPzzQv6+i4fwnsW5bRvnI4TAbLtSJYTvuh/dryq86gN3qSrSVJ0H+frxYD88gDr9ff2eDz5RWolirjmlgt61ihQi/sL30UUXg73WVFnIvJlevAg2R3BtSq3GwXWG61k/K7aFR3OfFfBVdW8Yq4aXDhiooOLHIOVNE1ser4XEUNqlN1HIoliQ4sMAdu40mfgJ9iVGU1oXCDq3IBY9miA+V1oIquieWtTuXfFw2qfE9jjtGB6EB0IDpQPR1Y2GuiH51GRQeiA9GB6EB0IDoQHSjRgbhpKNGieEF0IDoQHYgORAeiAzqwqG0aelMnv2l8jZWrIvJb9n4bO+hmAn7T+DHwG9yVoaPIpGdlZFSGPNbkWv/GHRUdiA5EB6ID0YGq4UAZ/37zK6XeCPz5TVXRdxQkbA78AqU/81L+BGq1TKjiL1eQxQUVz6ZMOWzJ1e+W6Y54cXQgOhAdiA5UyIEyrokVelaum/0J1aIif5PrZmEdmAR/gL8fdmH+Gu4GNxX+9nh1aA4rwHR4En4CF/dNwN+kqw3gd3Ch3yrBn/4NB39nrLqDv1neBe6BCXAQmNdzkJb5vZNKyCPsb7T9eVh/+B6U91r2huDmwrLPBBV+5/4XYX97XABpNSPib9Mto9c+AWpT8J63jCB/0/wy+Ox9wJ+R6tmjYH3MRy+XgpZwQxLuwVF9UXj412tTUvy3A3yWzxwHltO4Mq0L+CnX2xA8sl0OhPqgR8+CKirdczuBz5oBA8E2VBvD9mB9rP9YiIoORAeiA9GBCjqwqP15Im3HACJHg4u4G4FhsAS0B/9BkQPAxdljV1AHwwPggqmuBBfBQ+FG+Ar85MBrtgN1PXhuaWgALvB7gs/1/kYQtAOB50OE47XghsYF20V+CXAjMxLc0BXA/nATqHx4EX6EP+ENWAmCLKvnfeZ0sCzh+T7L8ih/x34L/AqPgPUvANPfgiXBP6F4/yHQChqCi7ybCDcwV0EuWRf9OQ4+gcMgPHcfwv1gGnwDd0E3qAXW3zq7GbkMjoCi0jmV+WTlHI5fQhsYBvq3IdwDBZDLI5KjogPRgehAdGCRdKCMH8XMwYTG0BZcWH3XGuQCexDkgwu/C5Q6Bu7IhAr/mdHXCO8NbgJc3OqA71rXhKCHCJySRFz0XXSVi6vPddFVLsKzwM2JcmFcJhMqzHvrJOzBRc/nugC66QhyUX0niTzD8dBwguN+0AmuADcu74GLdNBgAv8BNxMfwVRoCG6UBkBtOA5cbJVHy98O3Gj9DIuB6gWPZEKFL26w3k3FQ3B9AjMg5KkHbk7cUFlfzwe5gTgXGoOfeGwDlqklrALFpf/O+eUh6AUC+mEbu7lx86LWBdsyKjoQHYgOVHsHyrgmVnp9naAXRbnofQa+Iw5yQTVdFcBsA+hp2BZcTJeDO8HFbQcYBG5ERsEhMBI+h83Ad8FBpikXuvHgAqi+hcmZ0D+bhe+SuIcPU2HflbvZKYDm8DyMg0sgtJP5e13QfQQ+TiK+M3eBTpfrKeLbgfV5Bd6ELWBn8GP7v+B96A/WcQQsDuF5Ewn/CWpVSD/b+4rSGE64qCs9+AncCFjW7vAWfAFdwPJ6/hToD15/AcyFotL1qR68AfotG4NlfBh8vmX9FHaBdD8gGhUdiA5EB6ID5XEgLA7lubcq3+O75aZZBXRBDQu2G4GgSQRcqI6GYfAyuMiGhZVg5t+tr8NxD3DhHgLpxdnFV5l/+rle08gTaFt4MRP65yV9bTOSfYd+VEJvju3hFAjP+pmwfx4IyiewVhK5meOucCm0StLCpsF38NZLdoAtYDCsAC/AI7A5rA1hk0Bwnn8rP7tuTbygCKXL6OLutdZtYBJ24+DC78bIurlReRZWAr13w3A3FJXuZuIPcJPQOsHwNeCnPCfDMnA87Am9ICo6EB2IDkQHKujAorppeBdfXIzcCCgXxO3gGSM55OJ6Jvhu3E8G3ASkF/nliPsufRK4sO4Ii0G2PiDhFzgsOeHzw8bAxdpFMi03BHXB8m0Jlq8F+M7ZvFwAj4PwrCcJnwQNwU8jboOweLtRej9Ju4Oj9be8bjR2g6HgpuFQ8ON7330vDS7QnvN+F9dGEJ5H8G/5Dn4P6AhLgGUvSl7TLTnp5kfvpoM+joQJ4DW7gs9qCta3Q3IcwdHNS1HpX3DOTxP6gPVcFT6ETuCG5CGoA6+CHrjBUHtBy0wovkQHogPRgejAoudAGf9+46JfP3HBRcl39i5Qb8F2oNaGWzKhf15cbFxgXLDV6XB5JlT44oLvx90ufvfCsXAWqMdh+Uyo8MUFzHIUgIv6g+ACPwpcyIIeI3AuuAC+B11Aea2bCzcAlsmFbhC4uPqu/Sr4Elw0jwB1FPTMhAoX9CcIb5nEj+ToJiLoSQIu1kEXEPgMhoPluRa2Av27E9KyLB+Df47pC9YvW+uTUAAPJUefF/zpTPgj0Ec3IZbtfFAHwGiwvV6C1UAVld6Sc9azALzvIFBuwm4G8xkPN4LeKdti80wovkQHogPRgWroQBnXxGpYwwoWORpUQQMX/O1uGtzQREUHogPRgehAJTuwsNfERfXPE5XcTDG76EB0IDoQHYgORAeqvAMLe1dV5Q2qegX0OxEbVr1ixRJFB6ID0YHq70BcE0tow2hQCQbF09GB6EB0IDpQYxxY2Gti/PNEjelqsaLRgehAdCA6EB2omANx01Ax/+Ld0YHoQHQgOhAdqDEOxE1DjWnqWNHoQHQgOhAdiA5UzIG4aaiYf/Hu6EB0IDoQHYgO1BgH4qahxjR1rGh0IDoQHYgORAcq5kDcNFTMv3h3dCA6EB2IDkQHaowDcdNQY5o6VjQ6EB2IDkQHogMVcyBuGirmX7w7OhAdiA5EB6IDNcaBuGmoMU0dKxodiA5EB6ID0YGKORA3DRXzL94dHYgORAeiA9GBGuNA3DTUmKaOFY0ORAeiA9GB6EDFHIibhor5F++ODkQHogPRgehAjXEgbhpqTFPHikYHogPRgehAdKBiDsRNQ8X8i3dHB6ID0YHoQHSgxjgQNw01pqljRaMD0YHoQHQgOlAxB+KmoWL+xbujA9GB6EB0IDpQYxyIm4aq09RbUJRaVac4sSTRgehAdCA6EB2Y14FFbdOwJtXrAxfNW81KjR1HbkdXao7/93+2wy4wt5Lz7UZ+F1ZSnr3J51roWEn5ZWezEQl3ZSfGeHQgOhAdiA5EB0rtQP369cuzkJ5d6geU70I3JpWp3cls08rMMJVXZZa1C/lunMq7soOVWdbKLlvMLzoQHYgOLHQHyrkmVlq561ZaTtUjoz0o5nowC26Fr0AtAfvBSjAbBsJkCFqbwN7gff1CYnJckeNhsBiMgXsgaC8C7eFC2B/aws0wHdJyw3BqKuFQwp+CC3QjeAGGQ1BrAj2hDgyDQRBkeQ6HevBUSEyODTj6KYnHX+Am+A0qqlCeumT0GqTLswnxzuC59+FpCKpPoBc0hndCYnL0TzUHwqqg77bXNFB5cDCYl+XfF/RnMBSnEzj5MbSATvAr3Ag/gGU5DSbDnWCfMO4zb4eo6EB0IDoQHajqDpRzV3V2jnq5YOyYpLtI3QLNkviSHFsmYRfbS5OwBzcMZ4GLmOcuh8tAef/14MKuNgcXmrT6ErkCNoKG0BHSyieyczqBsGVxM9A8Sbc8LvSqFfhMFzW1P7iAqqXgOnABVMfBo5lQ4Z9AbiS8chJvzdFry6IuXJz9SYNltTx6qFzAD8mECl/acnDDoCynf0IK8vkrJhHzHh1OcDwXNkzi1ut/YLsFed/j4GbLdgnXEixWYzjr90eU/t4OdYygY0Bfgg4ikN1e4Vw8RgeiA9GBBe5AOdfESiunf0uvKXKReS6p7E8czwc/XVC/g59A9AEX/bCQEfy/3eESmAuzwPsWB7U3TAGvcUF0gQwLEsGM3GxcCSPAd/ejIa2diDybTkjCd3GcnoQ/4ui7Y+VC5ibGMqt7YfVMqLA8VxGemcRdaH0XrdYHF8etwbJuCW522kNFZHnOht+STO7n2CEJe7CPnQJ6a7qf5qh14DXQP/UyvJkJFfq7IWGvt6y7wtfJkcPfeo+Q3touI/9OLT7wKqeHJZfo72OwSRLX8x5JWK9Wg+z2Sk7HQ3QgOhAdqHkOhHeANaHm/tkhLRe5sPifRHgUnAcucpdBkIv+3BDh+CfMSeJ/cXwDJiVxD0NTYYNe44KXS2uR6IYgnX+u69LnbbM/si4K5fHTh7B4h0vCBsJyWMd0+QxPDReW81hcedqSZw/Qz5+gCwTPiyur9R0P6bIS/b/vfEmpPGVvmLrfoBunb5M0N2Lm2Qo2gwchKjoQHYgORAcSB1wga4pc7MNHzdb7DHgoqfyyHF9Mwi4YayRhDy5c+6fixxN2UVMPQzfw3XJBQh7HWlAa+VH+A6W5MHXNI4TPTMU3IjwjifuuuReE5+9I2IVbvQfW/0coSHABbQwVkeXRyyA/IfguifgJyFPghsHNxfYQNJJAV1gySWiXxI3aVtbJjUVBghulFaG0Oo4L9cPnptWByOGwDPjnpE3hbQjqT6An2Ac+gqjoQHQgOhAdSBwIi0uVNcS/38ycObO05fR7C/79W/xY3nfXl4CfMtQBF5KmUA98F/kBqE7QHbzuG3DRUncUHjJ/fvDjdMvhnxJ2hsHwGrQBF5lZ4JchTfOccnFvAZZFDQIXS9UadgL/hJCt3iS4qF0JLsAXwCTwOwnKhc4y+AnDZLgNwkZmA8JuZKyLnyw0B/O4H5rA8VAbXEy/hAGgT8XJ520LLvBe6wKerku6PG6gbgXLo+engou/8lOZbeA6+ArcBBwDlvVbmApu2q4C7/WcGzrLax1uh1/APLYCy/Mz2GY3QVreuzXsA+Yf1IfAU2B9rP8TkD5P9P/6wtPwLkRFB6ID0YEq40AZ18QqU+4FVpCF/aWP+VjRzci7wXzMP2ad2wE3DcXJjeFFxV0Qz0UHogPRgYXlwMJeE30HF7VwHBjOY39dOI+usU89kZqvDWeDm4O0/DTEDcXV0BJWg6joQHQgOhAdSDmQPXGmTlWNYPwoZoG0w8E8Ja+IJ/lnjc+KOBeTowPRgehAdGABOhDXxBLMXtgfxZRQvHg6OhAdiA5EB6IDC8yBhb0mxj9PLLCmjg+KDkQHogPRgehA9XYgbhqqTvvVrzpFiSWJDkQHogPRgejAvx1Y1DYNramif5/f5d9VrbQU/3XCjSott38yOpdg+jsm+xG3Lv7kMJf8Qt/WuU4Uk1aRsu9LvpanWTH5V+RU+yT/iuQR740ORAeiA9GB+ejAorZpmI5XQ2Hj+ejZO+Ttv69QmVqFzMbD3FSmrxP230Rok0qraLAiZffXHv57Bm0rWogi7vffbXAjFBUdiA5EB6IDVdSBRW3T4E8YCyD808nZtvuzOv8holwLn/8w0/rgAp5La5Hoz/CmQPhnm8N15rc5ZP+JwX9MqVVykedCOEn6+3AQoYF/xwoDEzlMy0oL0aUJ/ABvh4TkWI/j8rA4+O9A+A9LpZUuu59g5EH60430tdnh4srjtXrjZk0fs5VHgp/O6H+2/OTCNvEfjLJOafnvWGwBrdOJhC1zHjQE1Qasc0nyWXpnW5jvcpDWSkT8R7WCViTgPwYWFR2IDkQHogM4ULsGueC72CvARWBHuBDCgulCciq4yG4AJ0GQC9f/wEVxE7gY/NcKlfdfAObnBsE8XayDzMvf/XeFvrATbAlpuXD9CL+lE0sI+4wdoEfWdW5eboVLwYXxZFgPcqk/ifuAC2NFpBf+C45bgXW5HtIbrwOJ50Nj0Lv0ItyD+GHgQn0RdIKgrQn0ATcG/lnFP98EubjvBseCz+sMvaAkmefdcAFYnoMhtLV+2c5dQLnBOQGONBIVHYgORAeiA9XAgXL+vOTsHFW7gbSwSfD0NtDNQEp5hH036mIVdAqB9DtSNx/mpVzMXLDScmOS1vlE9k0nZIUt69JZaSG6LQH/L4dcctHLtVBeS3rd5Abre04SDgcXzL7gu/uyygV146ybDiG+fiqtHmHLkJYbhTzwmW52lBs0N2pBlnVYEjF8TTiRHN2Y5afS3Ow8C+HThtSpYoODs87qYZskzc2Om5ggNzJ1QiQeowPRgejAwnagnGtipRU7LC6VlmEVzuhnyjY3VT4XqDPgcfBPC91hFKh1Cg+Z1ya8TkvFPyAcPhVYlbALYvrTAxeZ2vAXqDlwfyb07xcXfq///t+nyp1iXrOTu61vKEfIcDMC70NBSKjgsR3390vlMYtw+POQ79YvganwNbi5GASqI7ybCRW+WNa3krjv+teEPkncg57aZmk9TuSXdEIpwp9kXfMxcTcNX8Ln4Ccjbnxs27Fg+0VFB6ID0YHoAA7UpE2D70jTi/k2xMOitT3h9Lv2NYgHfUfAj8MnJwm+8/c/SlIuYi4uA42UQ4dyz13luK8itwzlZv9MIFdBekNEtMwazR1+gvBGcufiHN0sqB3gbnjPCHLjYBsoF2vr/6oRZPoWmVDhP69tvhdA9qYnuaTEQ3uucEPzTNaVfmJxKUwHn/kfSG9ObEs3Dm2z0olGRQeiA9GBmu3AorZpOJHmbAJbgnXz3auLo++8b4Ib4HVwE+C7Sc+pCXAM/ADrgh9THw63wy3gIvMh+P0G83dB2QpcgNeCs8B3qua7LPQGn30m5EPQ8wRGJBEXV/8UMjmJpw+bE+kCPud36Aq+Qx8JeXAweP/a4PcBLNsT4D354GbIxXI32BoGgM/xOwDWwU8DHgQ/3v8f9IPitBknt4E28Ce4yQp1uZ9wX9gIvgE3EPql3oGTwHq0A++vD5Z3Cuj7uTAO1oMv4FS4Em6Ea8HNiBu+1eBh0APLoie204rwLXh9Wv8lshfYXnoY9BQB21r/9OIu+AmCLIttPxZmh8R4jA5EB6ID0YFq4EAl//2mFlVuDS602WpBQqvsxFR8JcLNUvF0sBERF0QXqNJqby7sUNqLq8F1bpb0NltuEkx3gc6lxiR6vk6uk8k5vS+rfG6u9upTioz6ck29UlwXL4kORAeiAwvUgUpeExdo2RfIw6JBC8TmmvIQPw3xU5XNc1TYTctW4J8r/FQq18aS5KjoQHQgOrDwHFjYa2LdhVf1+OQq5EBHytK8iPL4p46fizhX3ZJrU+AB4AahFsyFIM+Z5p8qHoKiPvngVFR0IDoQHaiZDsRNQ81s9+xaL5GdkIovSXhR2TS8mapXdtDveQzJTozx6EB0IDoQHfjHgbhp+MeLmhwKv26oyR7EukcHogPRgehACQ74kWxUdCA6EB2IDkQHogPRgRIdiJuGEi2KF0QHogPRgehAdCA6oANx0xD7QXQgOhAdiA5EB6IDpXKgOnynYcbC/olJqZyMF0UHogPRgehAdGD+OzBj/j8iPiE6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdCA6EB2IDkQHogPRgehAdKAYB/4ffeRvbLZiFvMAAAAASUVORK5CYII=", - "text/plain": [ - "" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Image(filename='../../../docs/source/_figures/remote_3.png')" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "from torch_geometric.loader import RAGQueryLoader" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [], - "source": [ - "query_loader = RAGQueryLoader(\n", - " data=(feature_store, graph_store), # Remote Rag Graph Store and Feature Store\n", - " # Arguments to pass into the seed node/edge retrieval methods for the FeatureStore.\n", - " # In this case, it's k for the KNN on the nodes and edges.\n", - " seed_nodes_kwargs={\"k_nodes\": 10}, seed_edges_kwargs={\"k_edges\": 10}, \n", - " # Arguments to pass into the GraphStore's Neighbor sampling method.\n", - " # In this case, the GraphStore implements a NeighborLoader, so it takes the same arguments.\n", - " sampler_kwargs={\"num_neighbors\": [40]*3},\n", - " # Arguments to pass into the FeatureStore's feature loading method.\n", - " loader_kwargs={},\n", - " # An optional local transform that can be applied on the returned subgraph.\n", - " local_filter=None,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To make better sense of this loader's arguments, let's take a closer look at the retrieval process for a remote backend:" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SVG(filename=\"media/remote_backend.svg\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As we see here, there are 3 important steps to any remote backend procedure for graphs:\n", - "1. Retrieve the seed nodes and edges to begin our retrieval process from.\n", - "2. Traverse the graph neighborhood of the seed nodes/edges to gather local context.\n", - "3. Fetch the features associated with the subgraphs obtained from the traversal.\n", - "\n", - "We can see that our Query Loader construction allows us to specify unique hyperparameters for each unique step in this retrieval." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can submit our queries to the remote backend to retrieve our subgraphs:" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/10 [00:00" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "svg_file_path = './media/flowchart.svg'\n", - "SVG(svg_file_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Graph RAG as shown in the diagram above follows the following order of operations:\n", - "\n", - "0. To start, not pictured here, there must exist a large knowledge graph that exists as a source of truth. The nodes and edges of this knowledge graph \n", - "\n", - "During inference time, RAG implementations that follow this architecture are composed of the following steps:\n", - "\n", - "1. Tokenize and encode the query using the LLM Encoder\n", - "2. Retrieve a subgraph of the larger knowledge graph (KG) relevant to the query and encode it using a GNN\n", - "3. Jointly embed the GNN embedding with the LLM embedding\n", - "4. Utilize LLM Decoder to decode joint embedding and generate a response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Each notebook that follows will highlight a different component of this pipeline." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pyg-local-dev", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From a597870703e0ad8b20e63bea6441e35af65cd415 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:54:12 -0700 Subject: [PATCH 720/752] readme in llm_plus_gnn --- examples/llm_plus_gnn/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 33d9c22aa56b..fd93a1d6c5f9 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,5 +1,12 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | +| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | +| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided | From e5e5453c883467cfd5bdb185052bbf515d0d12ae Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:56:13 -0700 Subject: [PATCH 721/752] fix multihop 2 --- examples/llm_plus_gnn/multihop/rag_generate_multihop.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index 6c53ca18484b..8340ef3f963b 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -76,7 +76,8 @@ def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, # %% subgs = [] -for subg in tqdm.tqdm(query_loader.query(q) for q in questions): +for q in tqdm.tqdm(questions): + subg = query_loader.query(q) subg['question'] = q subgs.append(subg) From a959118cae7ad3187ffa5bf98f0ab88b53cba0d5 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 14:59:52 -0700 Subject: [PATCH 722/752] linting --- examples/llm_plus_gnn/README.md | 20 ++++++++++---------- examples/llm_plus_gnn/rag_generate.py | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index fd93a1d6c5f9..e34e83645ffd 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,12 +1,12 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | -| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | -| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | -| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | -| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | -| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided | +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | +| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | +| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided | diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 18b4a769694a..7c1d7f6b5ff9 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -129,5 +129,6 @@ def check_retrieval_recall(subg: Data, ground_truth: Data): subg['question'] = questions[i] subg['label'] = ds[i]['label'] -pd.DataFrame.from_dict(retrieval_stats).to_csv(args.out_file.split('.')[0] + '_metadata.csv') +pd.DataFrame.from_dict(retrieval_stats).to_csv( + args.out_file.split('.')[0] + '_metadata.csv') torch.save(subgs, args.out_file) From 0ef2bbc1070cf654cd2628de30a32f1ad31f5df3 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 15:03:56 -0700 Subject: [PATCH 723/752] close parenthesis --- examples/llm_plus_gnn/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index e34e83645ffd..5e13721c1dd3 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,12 +1,12 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | -| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | -| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | -| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | -| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | -| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided | +| Example | Description | +| ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | +| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | +| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | From 6fab6a5d61204f10ed1c0f432938c262eff34f90 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 15:45:11 -0700 Subject: [PATCH 724/752] fix multihop 3 --- examples/llm_plus_gnn/multihop/rag_generate_multihop.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py index 8340ef3f963b..41c2e82357e0 100644 --- a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop/rag_generate_multihop.py @@ -35,6 +35,7 @@ # %% df = pd.read_csv('wikimultihopqa_cleaned.csv') questions = df['question'][:args.num_samples] +labels = df['answer'][:args.num_samples] # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -76,9 +77,10 @@ def apply_retrieval_via_pcst(graph: Data, query: str, topk: int = 3, # %% subgs = [] -for q in tqdm.tqdm(questions): +for q, l in tqdm.tqdm(zip(questions, labels)): subg = query_loader.query(q) subg['question'] = q + subg['label'] = l subgs.append(subg) torch.save(subgs, 'subg_results.pt') From d9ef859fedd75560ee3f8b118769e656a03df3d9 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 16:19:11 -0700 Subject: [PATCH 725/752] fix typo in example --- docs/source/_figures/multihop_example.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_figures/multihop_example.svg b/docs/source/_figures/multihop_example.svg index 42e2eaf36afa..e88ee4e6e941 100644 --- a/docs/source/_figures/multihop_example.svg +++ b/docs/source/_figures/multihop_example.svg @@ -1 +1 @@ - + \ No newline at end of file From 352c4ab183bf3c9b19736fccfb0244c82665ec8f Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 17:05:56 -0700 Subject: [PATCH 726/752] multihop to multihop rag --- docs/source/advanced/rag.rst | 2 +- examples/llm_plus_gnn/README.md | 2 +- .../{multihop => multihop_rag}/multihop_download.sh | 0 .../{multihop => multihop_rag}/multihop_preprocess.py | 0 .../{multihop => multihop_rag}/rag_generate_multihop.py | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename examples/llm_plus_gnn/{multihop => multihop_rag}/multihop_download.sh (100%) rename examples/llm_plus_gnn/{multihop => multihop_rag}/multihop_preprocess.py (100%) rename examples/llm_plus_gnn/{multihop => multihop_rag}/rag_generate_multihop.py (100%) diff --git a/docs/source/advanced/rag.rst b/docs/source/advanced/rag.rst index d20623f87dab..2d8513fd56e4 100644 --- a/docs/source/advanced/rag.rst +++ b/docs/source/advanced/rag.rst @@ -313,7 +313,7 @@ Building a Multi-Hop QA Dataset To see an example of encoding a large knowledge graph starting from an existing set of triplets, check out the multi-hop example in -`examples/llm_plus_gnn/multihop`. +`examples/llm_plus_gnn/multihop_rag`. Question: How do we extract a contextual subgraph for a given query? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 5e13721c1dd3..3e8824bcb8b1 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -4,7 +4,7 @@ | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | | [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | -| [`multihop/`](./multihop/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | | [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | | [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | | [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | diff --git a/examples/llm_plus_gnn/multihop/multihop_download.sh b/examples/llm_plus_gnn/multihop_rag/multihop_download.sh similarity index 100% rename from examples/llm_plus_gnn/multihop/multihop_download.sh rename to examples/llm_plus_gnn/multihop_rag/multihop_download.sh diff --git a/examples/llm_plus_gnn/multihop/multihop_preprocess.py b/examples/llm_plus_gnn/multihop_rag/multihop_preprocess.py similarity index 100% rename from examples/llm_plus_gnn/multihop/multihop_preprocess.py rename to examples/llm_plus_gnn/multihop_rag/multihop_preprocess.py diff --git a/examples/llm_plus_gnn/multihop/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py similarity index 100% rename from examples/llm_plus_gnn/multihop/rag_generate_multihop.py rename to examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py From 97ab60a33ae671aa72fd54da9c5cc02753d102aa Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Thu, 29 Aug 2024 17:17:09 -0700 Subject: [PATCH 727/752] linting --- docs/source/_figures/multihop_example.svg | 2 +- examples/llm_plus_gnn/README.md | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/_figures/multihop_example.svg b/docs/source/_figures/multihop_example.svg index e88ee4e6e941..4925dcb9713d 100644 --- a/docs/source/_figures/multihop_example.svg +++ b/docs/source/_figures/multihop_example.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 3e8824bcb8b1..73035cae6cc6 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,12 +1,12 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | -| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | -| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | -| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | -| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | -| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | +| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | +| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | From ab53471c0e96093dd3ac96b3ff8fd57493208f80 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 13:50:14 -0700 Subject: [PATCH 728/752] typo --- examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index c341877dc480..7be6d6c8e876 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -17,7 +17,7 @@ from torch_geometric.profile.nvtx import nvtxit sys.path.append('..') -from profiling_utils import create_remote_backend_from_triplets # noqa: E402 +from rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 from rag_feature_store import SentenceTransformerFeatureStore # noqa: E402 from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 From 99e715d8dc7d8f74e407ab16d0b16f8ec1e52c90 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 13:50:35 -0700 Subject: [PATCH 729/752] fix bug where force flag wasnt being acknoledged for LLM results --- examples/llm_plus_gnn/g_retriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 3585debb32e1..674cb5729b8c 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -349,7 +349,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], else: pure_llm = LLM(model_name="llama2-7b", num_params=7) - if not path.exists(root_dir + "/pure_llm_model_log.pt"): + if force or not path.exists(root_dir + "/pure_llm_model_log.pt"): model_log["pure_llm"] = dict() pure_preds = [] @@ -371,7 +371,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], torch.load(root_dir+"/pure_llm_model_log.pt") # LORA - if not path.exists(root_dir + "/tuned_llm_model_log.pt"): + if force or not path.exists(root_dir + "/tuned_llm_model_log.pt"): model_log["tuned_llm"] = dict() since = time.time() gc.collect() From 3eea91286dd03330568d24857f8b70da0c7384f5 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 13:56:29 -0700 Subject: [PATCH 730/752] restore patch from rishis commits --- torch_geometric/datasets/web_qsp_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5d2518f448ad..e68e4d42c7d5 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -88,8 +88,9 @@ def retrieval_via_pcst( indices = e_prizes == topk_e_values[k] value = min((topk_e - k) / sum(indices), last_topk_e_value - c) e_prizes[indices] = value - last_topk_e_value = value - # cost_e = max(min(cost_e, e_prizes.max().item()-c), 0) + last_topk_e_value = value * (1 - c) + # reduce the cost of the edges such that at least one edge is selected + cost_e = min(cost_e, e_prizes.max().item() * (1 - c / 2)) else: e_prizes = torch.zeros(graph.num_edges) From 6560566d85945237703ccd9965a1125740f81cbd Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 14:58:06 -0700 Subject: [PATCH 731/752] add verbosity flag for webqsp --- examples/llm_plus_gnn/benchmark_model_archs_rag.py | 2 +- examples/llm_plus_gnn/rag_generate.py | 2 +- torch_geometric/datasets/web_qsp_dataset.py | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs_rag.py b/examples/llm_plus_gnn/benchmark_model_archs_rag.py index ba30419552e1..20a79504bbd9 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs_rag.py +++ b/examples/llm_plus_gnn/benchmark_model_archs_rag.py @@ -36,7 +36,7 @@ # %% if not args.dataset_path: - ds = WebQSPDataset('benchmark_archs') + ds = WebQSPDataset('benchmark_archs', verbose=True) else: # We just assume that the size of the dataset accomodates the # train/val/test split, because checking may be expensive. diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 7c1d7f6b5ff9..15f90d57b014 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -28,7 +28,7 @@ args = parser.parse_args() # %% -ds = WebQSPDataset("dataset", limit=args.num_samples) +ds = WebQSPDataset("dataset", limit=args.num_samples, verbose=True) # %% triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index e68e4d42c7d5..b03d9e92d0d0 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -183,6 +183,7 @@ def __init__( force_reload: bool = False, limit: int = -1, include_pcst: bool = True, + verbose: bool = False, ) -> None: """Construct a WebQSPDataset. @@ -194,12 +195,14 @@ def __init__( Defaults to -1 to construct all samples. include_pcst (bool, optional): Whether to include PCST step (See GRetriever paper). Defaults to True. + verbose (bool, optional): Whether to print output. Defaults to False. """ self.limit = limit self.include_pcst = include_pcst self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self._check_dependencies() + self.verbose = verbose super().__init__(root, None, None, force_reload=force_reload) self._load_raw_data() self.load(self.processed_paths[0]) @@ -306,9 +309,10 @@ def _retrieve_subgraphs(self) -> None: textual_edges = self.textual_edges graph_gen = get_features_for_triplets_groups( self.indexer, (ds['graph'] for ds in self.raw_dataset), - pre_transform=preprocess_triplet) + pre_transform=preprocess_triplet, verbose=self.verbose) - for index in tqdm(range(len(self.raw_dataset)), disable=True): + for index in tqdm(range(len(self.raw_dataset)), + disable=not self.verbose): data_i = self.raw_dataset[index] graph = next(graph_gen) textual_nodes = self.textual_nodes.iloc[ From 9852a76550a6443130d9fcb83e475acac8368da9 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 15:25:51 -0700 Subject: [PATCH 732/752] fix force reload for webqsp and set to default --- examples/llm_plus_gnn/benchmark_model_archs_rag.py | 2 +- examples/llm_plus_gnn/rag_generate.py | 3 ++- torch_geometric/datasets/web_qsp_dataset.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/benchmark_model_archs_rag.py b/examples/llm_plus_gnn/benchmark_model_archs_rag.py index 20a79504bbd9..b42568d83687 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs_rag.py +++ b/examples/llm_plus_gnn/benchmark_model_archs_rag.py @@ -36,7 +36,7 @@ # %% if not args.dataset_path: - ds = WebQSPDataset('benchmark_archs', verbose=True) + ds = WebQSPDataset('benchmark_archs', verbose=True, force_reload=True) else: # We just assume that the size of the dataset accomodates the # train/val/test split, because checking may be expensive. diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/rag_generate.py index 15f90d57b014..f73eadb983b8 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/rag_generate.py @@ -28,7 +28,8 @@ args = parser.parse_args() # %% -ds = WebQSPDataset("dataset", limit=args.num_samples, verbose=True) +ds = WebQSPDataset("dataset", limit=args.num_samples, verbose=True, + force_reload=True) # %% triplets = chain.from_iterable(d['graph'] for d in ds.raw_dataset) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index b03d9e92d0d0..ba3acfbea370 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -203,6 +203,7 @@ def __init__( "cuda" if torch.cuda.is_available() else "cpu") self._check_dependencies() self.verbose = verbose + self.force_reload = force_reload super().__init__(root, None, None, force_reload=force_reload) self._load_raw_data() self.load(self.processed_paths[0]) @@ -344,7 +345,7 @@ def process(self) -> None: self.model = SentenceTransformer( 'sentence-transformers/all-roberta-large-v1').to(self.device) self.model.eval() - if not os.path.exists(self.processed_paths[-1]): + if self.force_reload or not os.path.exists(self.processed_paths[-1]): print("Encoding graph...") self._build_graph() else: From f41e8d6281304cf445f5e971b65ce803ac25e8b8 Mon Sep 17 00:00:00 2001 From: Zack Aristei Date: Fri, 30 Aug 2024 18:41:20 -0700 Subject: [PATCH 733/752] update llama2 url --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 674cb5729b8c..ef99962ce186 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -347,7 +347,8 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], num_params=1, ) else: - pure_llm = LLM(model_name="llama2-7b", num_params=7) + pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", + num_params=7) if force or not path.exists(root_dir + "/pure_llm_model_log.pt"): model_log["pure_llm"] = dict() From 60ef416292f2a0dc286baa0917e4504bad3136d5 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Sun, 1 Sep 2024 21:26:29 -0400 Subject: [PATCH 734/752] restore webqsp as default --- examples/llm_plus_gnn/g_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index ef99962ce186..0410e13853bf 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -694,7 +694,8 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, print("E2E tme minus Prep Time =", e2e_time - prep_time, "seconds") else: gnn_llm_eval_outs = torch.load("gnn_llm_eval_outs.pt") - dataset = None + # use webqsp dataset as default for minimal demo + dataset = WebQSPDataset() print("Here's a demo showcasing how GNN reduces LLM hallucinations:") minimal_demo(gnn_llm_eval_outs, dataset, args.lr, args.epochs, args.batch_size, args.eval_batch_size, get_loss, From d06a285aa18035659a3e4246194a430b68e5832a Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Sun, 1 Sep 2024 21:41:23 -0400 Subject: [PATCH 735/752] fix skip untuned arg issues --- examples/llm_plus_gnn/g_retriever.py | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index 0410e13853bf..f6abe4d30bba 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -520,28 +520,29 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, for i, batch in enumerate(tqdm(loader)): question = batch.question correct_answer = batch.label - if skip_pretrained_LLM: - pure_llm_pred = None - pure_llm_hallucinates = False - else: + + gnn_llm_pred = gnn_llm_preds[i * eval_batch_size:(i + 1) * + eval_batch_size] + gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, + correct_answer) + gnn_save_list += [ + tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates) + ] + + if not skip_pretrained_LLM: # GNN+LLM only using 32 tokens to answer. # Allow more output tokens for untrained LLM pure_llm_pred = pure_llm.inference(batch.question, batch.desc, max_tokens=256) pure_llm_hallucinates = detect_hallucinate( pure_llm_pred, correct_answer) + else: + pure_llm_pred = [''] * len(gnn_llm_hallucinates) + pure_llm_hallucinates = [False]* len(gnn_llm_hallucinates) untuned_llm_save_list += [ tup for tup in zip(pure_llm_pred, pure_llm_hallucinates) ] - gnn_llm_pred = gnn_llm_preds[i * eval_batch_size:(i + 1) * - eval_batch_size] - gnn_llm_hallucinates = detect_hallucinate(gnn_llm_pred, - correct_answer) - gnn_save_list += [ - tup for tup in zip(gnn_llm_pred, gnn_llm_hallucinates) - ] - for gnn_llm_hal, pure_llm_hal in zip(gnn_llm_hallucinates, pure_llm_hallucinates): if gnn_llm_hal == "skip" or pure_llm_hal == "skip": # noqa @@ -606,21 +607,20 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, untuned_llm_pred, untuned_llm_hallucinates = list( zip(*untuned_llm_save_list[i * eval_batch_size:(i + 1) * eval_batch_size])) - if gnn_llm_hallucinates == "skip" or untuned_llm_hallucinates == "skip": # noqa - continue pure_llm_pred = pure_llm_preds[i * eval_batch_size:(i + 1) * eval_batch_size] pure_llm_hallucinates = detect_hallucinate(pure_llm_pred, correct_answer) for j in range(len(gnn_llm_pred)): + if skip_pretrained_LLM: + # we did not check the untrained LLM, so do not decide to demo + # based on this. + # HACK + untuned_llm_hallucinates = {j: True} if gnn_llm_hallucinates[j] == "skip" or untuned_llm_hallucinates[ j] == "skip" or pure_llm_hallucinates[j] == "skip": continue trained_llm_hallucin_sum += int(pure_llm_hallucinates[j]) - if skip_pretrained_LLM: - # we did not check the untrained LLM, so do not decide to demo - # based on this. - untuned_llm_hallucinates = True if untuned_llm_hallucinates[j] and pure_llm_hallucinates[ j] and not gnn_llm_hallucinates[j]: # noqa final_prnt_str += "Prompt: '" + question[j] + "'\n" From f88be6b7662ee562b0b4e43fc11b443e3224e833 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Wed, 4 Sep 2024 10:40:59 -0400 Subject: [PATCH 736/752] reform file structure and add more documentation --- examples/llm_plus_gnn/README.md | 6 +----- examples/llm_plus_gnn/g_retriever_utils/README.md | 11 +++++++++++ .../benchmark_model_archs_rag.py | 8 ++++++-- .../{ => g_retriever_utils}/rag_backend_utils.py | 0 .../{ => g_retriever_utils}/rag_feature_store.py | 0 .../{ => g_retriever_utils}/rag_generate.py | 3 ++- .../{ => g_retriever_utils}/rag_graph_store.py | 0 examples/llm_plus_gnn/multihop_rag/README.md | 9 +++++++++ .../multihop_rag/rag_generate_multihop.py | 6 +++--- examples/llm_plus_gnn/nvtx_examples/README.md | 7 +++++++ .../nvtx_examples/nvtx_rag_backend_example.py | 6 +++--- 11 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 examples/llm_plus_gnn/g_retriever_utils/README.md rename examples/llm_plus_gnn/{ => g_retriever_utils}/benchmark_model_archs_rag.py (92%) rename examples/llm_plus_gnn/{ => g_retriever_utils}/rag_backend_utils.py (100%) rename examples/llm_plus_gnn/{ => g_retriever_utils}/rag_feature_store.py (100%) rename examples/llm_plus_gnn/{ => g_retriever_utils}/rag_generate.py (95%) rename examples/llm_plus_gnn/{ => g_retriever_utils}/rag_graph_store.py (100%) create mode 100644 examples/llm_plus_gnn/multihop_rag/README.md create mode 100644 examples/llm_plus_gnn/nvtx_examples/README.md diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 73035cae6cc6..524c0d1b92c0 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -3,10 +3,6 @@ | Example | Description | | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | -| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | +| [`g_retriever_utils/`](./g_retriever_utils/) | Contains multiple scripts for benchmarking GRetriever's architecture and evaluating different retrieval methods. | | [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | | [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | -| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | -| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | -| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | diff --git a/examples/llm_plus_gnn/g_retriever_utils/README.md b/examples/llm_plus_gnn/g_retriever_utils/README.md new file mode 100644 index 000000000000..bdda5358344c --- /dev/null +++ b/examples/llm_plus_gnn/g_retriever_utils/README.md @@ -0,0 +1,11 @@ +# Examples for LLM and GNN co-training + +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`rag_feature_store.py`](./rag_feature_store.py) | A Proof of Concept Implementation of a RAG enabled FeatureStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_graph_store.py`](./rag_graph_store.py) | A Proof of Concept Implementation of a RAG enabled GraphStore that can serve as a starting point for implementing a custom RAG Remote Backend | +| [`rag_backend_utils.py`](./rag_backend_utils.py) | Utility functions used for loading a series of Knowledge Graph Triplets into the Remote Backend defined by a FeatureStore and GraphStore | +| [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | +| [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | + +NOTE: Evaluating performance on GRetriever with smaller sample sizes may result in subpar performance. It is not unusual for the fine-tuned model/LLM to perform worse than an untrained LLM on very small sample sizes. \ No newline at end of file diff --git a/examples/llm_plus_gnn/benchmark_model_archs_rag.py b/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py similarity index 92% rename from examples/llm_plus_gnn/benchmark_model_archs_rag.py rename to examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py index b42568d83687..b53f26ec6271 100644 --- a/examples/llm_plus_gnn/benchmark_model_archs_rag.py +++ b/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py @@ -5,13 +5,17 @@ import argparse import torch -from g_retriever import benchmark_models, get_loss, inference_step +import sys from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever +sys.path.append('..') +from g_retriever import benchmark_models, get_loss, inference_step # noqa: E402 + # %% -parser = argparse.ArgumentParser(description="Benchmarker for GRetriever") +parser = argparse.ArgumentParser(description="""Benchmarker for GRetriever +NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""") parser.add_argument("--hidden_channels", type=int, default=1024) parser.add_argument("--learning_rate", type=float, default=1e-5) parser.add_argument("--epochs", type=int, default=2) diff --git a/examples/llm_plus_gnn/rag_backend_utils.py b/examples/llm_plus_gnn/g_retriever_utils/rag_backend_utils.py similarity index 100% rename from examples/llm_plus_gnn/rag_backend_utils.py rename to examples/llm_plus_gnn/g_retriever_utils/rag_backend_utils.py diff --git a/examples/llm_plus_gnn/rag_feature_store.py b/examples/llm_plus_gnn/g_retriever_utils/rag_feature_store.py similarity index 100% rename from examples/llm_plus_gnn/rag_feature_store.py rename to examples/llm_plus_gnn/g_retriever_utils/rag_feature_store.py diff --git a/examples/llm_plus_gnn/rag_generate.py b/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py similarity index 95% rename from examples/llm_plus_gnn/rag_generate.py rename to examples/llm_plus_gnn/g_retriever_utils/rag_generate.py index f73eadb983b8..8c7591bd28a6 100644 --- a/examples/llm_plus_gnn/rag_generate.py +++ b/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py @@ -20,7 +20,8 @@ from torch_geometric.nn.nlp import SentenceTransformer # %% -parser = argparse.ArgumentParser(description="Generate new WebQSP subgraphs") +parser = argparse.ArgumentParser(description="""Generate new WebQSP subgraphs +NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""") # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") parser.add_argument("--num_samples", type=int, default=4700) diff --git a/examples/llm_plus_gnn/rag_graph_store.py b/examples/llm_plus_gnn/g_retriever_utils/rag_graph_store.py similarity index 100% rename from examples/llm_plus_gnn/rag_graph_store.py rename to examples/llm_plus_gnn/g_retriever_utils/rag_graph_store.py diff --git a/examples/llm_plus_gnn/multihop_rag/README.md b/examples/llm_plus_gnn/multihop_rag/README.md new file mode 100644 index 000000000000..491ac2c9bafb --- /dev/null +++ b/examples/llm_plus_gnn/multihop_rag/README.md @@ -0,0 +1,9 @@ +# Examples for LLM and GNN co-training + +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`multihop_download.sh`](./multihop_download.sh) | Downloads all the components of the multihop dataset. | +| [`multihop_preprocess.py`](./multihop_preprocess.py) | Preprocesses the dataset to pair questions/answers with components in the knowledge graph. Contains documentation to describe the process. | +| [`rag_generate_multihop.py`](./rag_generate_multihop.py) | Utilizes the sample remote backend in [`g_retriever_utils`](../g_retriever_utils/) to generate subgraphs for the multihop dataset. | + +NOTE: Performance of GRetriever on this dataset has not been evaluated. \ No newline at end of file diff --git a/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py index 41c2e82357e0..4f853406f42b 100644 --- a/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py @@ -17,10 +17,10 @@ sys.path.append('..') -from rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 -from rag_feature_store import \ +from g_retriever_utils.rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 +from g_retriever_utils.rag_feature_store import \ SentenceTransformerApproxFeatureStore # noqa: E402 -from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 +from g_retriever_utils.rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 # %% parser = argparse.ArgumentParser( diff --git a/examples/llm_plus_gnn/nvtx_examples/README.md b/examples/llm_plus_gnn/nvtx_examples/README.md new file mode 100644 index 000000000000..10e0f495a24c --- /dev/null +++ b/examples/llm_plus_gnn/nvtx_examples/README.md @@ -0,0 +1,7 @@ +# Examples for LLM and GNN co-training + +| Example | Description | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`nvtx_run.sh`](./nvtx_run.sh) | Runs nsys profiler on a given Python file that contains NVTX calls. | +| [`nvtx_rag_backend_example.py`](./nvtx_rag_backend_example.py) | Example script for nsys profiling a RAG Backend such as that used in [`rag_generate.py`](../g_retriever_utils/rag_generate.py). | +| [`nvtx_webqsp_example.py`](./nvtx_webqsp_example.py) | Example script for nsys profiling the WebQSP dataset. | diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index 7be6d6c8e876..fe1cae4d608f 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -17,9 +17,9 @@ from torch_geometric.profile.nvtx import nvtxit sys.path.append('..') -from rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 -from rag_feature_store import SentenceTransformerFeatureStore # noqa: E402 -from rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 +from g_retriever_utils.rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 +from g_retriever_utils.rag_feature_store import SentenceTransformerFeatureStore # noqa: E402 +from g_retriever_utils.rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 # %% # Patch FeatureStore and GraphStore From 7f5d8f1625aa12a7f2562b3764e3c6b5efff3721 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 01:19:19 +0000 Subject: [PATCH 737/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm_plus_gnn/README.md | 12 ++++++------ examples/llm_plus_gnn/g_retriever.py | 4 ++-- examples/llm_plus_gnn/g_retriever_utils/README.md | 2 +- .../g_retriever_utils/benchmark_model_archs_rag.py | 11 ++++++++--- .../llm_plus_gnn/g_retriever_utils/rag_generate.py | 5 +++-- examples/llm_plus_gnn/multihop_rag/README.md | 12 ++++++------ .../multihop_rag/rag_generate_multihop.py | 6 ++++-- examples/llm_plus_gnn/nvtx_examples/README.md | 10 +++++----- .../nvtx_examples/nvtx_rag_backend_example.py | 9 ++++++--- test/nn/models/test_g_retriever.py | 3 ++- torch_geometric/datasets/web_qsp_dataset.py | 6 +++--- torch_geometric/nn/models/g_retriever.py | 3 ++- 12 files changed, 48 insertions(+), 35 deletions(-) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm_plus_gnn/README.md index 524c0d1b92c0..728259a9be40 100644 --- a/examples/llm_plus_gnn/README.md +++ b/examples/llm_plus_gnn/README.md @@ -1,8 +1,8 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | -| [`g_retriever_utils/`](./g_retriever_utils/) | Contains multiple scripts for benchmarking GRetriever's architecture and evaluating different retrieval methods. | -| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | -| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| Example | Description | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training LLAMA2 with GAT for answering questions based on knowledge graph information | +| [`g_retriever_utils/`](./g_retriever_utils/) | Contains multiple scripts for benchmarking GRetriever's architecture and evaluating different retrieval methods. | +| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py index f6abe4d30bba..325f5ef023d9 100644 --- a/examples/llm_plus_gnn/g_retriever.py +++ b/examples/llm_plus_gnn/g_retriever.py @@ -347,7 +347,7 @@ def benchmark_models(models: List[Type[nn.Module]], model_names: List[str], num_params=1, ) else: - pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", + pure_llm = LLM(model_name="meta-llama/Llama-2-7b-chat-hf", num_params=7) if force or not path.exists(root_dir + "/pure_llm_model_log.pt"): @@ -538,7 +538,7 @@ def minimal_demo(gnn_llm_eval_outs, dataset, lr, epochs, batch_size, pure_llm_pred, correct_answer) else: pure_llm_pred = [''] * len(gnn_llm_hallucinates) - pure_llm_hallucinates = [False]* len(gnn_llm_hallucinates) + pure_llm_hallucinates = [False] * len(gnn_llm_hallucinates) untuned_llm_save_list += [ tup for tup in zip(pure_llm_pred, pure_llm_hallucinates) ] diff --git a/examples/llm_plus_gnn/g_retriever_utils/README.md b/examples/llm_plus_gnn/g_retriever_utils/README.md index bdda5358344c..e072e6746b7c 100644 --- a/examples/llm_plus_gnn/g_retriever_utils/README.md +++ b/examples/llm_plus_gnn/g_retriever_utils/README.md @@ -8,4 +8,4 @@ | [`rag_generate.py`](./rag_generate.py) | Script for generating a unique set of subgraphs from the WebQSP dataset using a custom defined retrieval algorithm (defaults to the FeatureStore and GraphStore provided) | | [`benchmark_model_archs_rag.py`](./benchmark_model_archs_rag.py) | Script for running a GNN/LLM benchmark on GRetriever while grid searching relevent architecture parameters and datasets. | -NOTE: Evaluating performance on GRetriever with smaller sample sizes may result in subpar performance. It is not unusual for the fine-tuned model/LLM to perform worse than an untrained LLM on very small sample sizes. \ No newline at end of file +NOTE: Evaluating performance on GRetriever with smaller sample sizes may result in subpar performance. It is not unusual for the fine-tuned model/LLM to perform worse than an untrained LLM on very small sample sizes. diff --git a/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py b/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py index b53f26ec6271..76148cfc09e5 100644 --- a/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py +++ b/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py @@ -3,19 +3,24 @@ """ # %% import argparse +import sys import torch -import sys from torch_geometric.datasets import WebQSPDataset from torch_geometric.nn.models import GAT, MLP, GRetriever sys.path.append('..') -from g_retriever import benchmark_models, get_loss, inference_step # noqa: E402 +from g_retriever import ( # noqa: E402 + benchmark_models, + get_loss, + inference_step, +) # %% parser = argparse.ArgumentParser(description="""Benchmarker for GRetriever -NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""") +NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""" + ) parser.add_argument("--hidden_channels", type=int, default=1024) parser.add_argument("--learning_rate", type=float, default=1e-5) parser.add_argument("--epochs", type=int, default=2) diff --git a/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py b/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py index 8c7591bd28a6..c6895b453b0c 100644 --- a/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py +++ b/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py @@ -21,7 +21,8 @@ # %% parser = argparse.ArgumentParser(description="""Generate new WebQSP subgraphs -NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""") +NOTE: Evaluating with smaller samples may result in poorer performance for the trained models compared to untrained models.""" + ) # TODO: Add more arguments for configuring rag params parser.add_argument("--use_pcst", action="store_true") parser.add_argument("--num_samples", type=int, default=4700) @@ -29,7 +30,7 @@ args = parser.parse_args() # %% -ds = WebQSPDataset("dataset", limit=args.num_samples, verbose=True, +ds = WebQSPDataset("dataset", limit=args.num_samples, verbose=True, force_reload=True) # %% diff --git a/examples/llm_plus_gnn/multihop_rag/README.md b/examples/llm_plus_gnn/multihop_rag/README.md index 491ac2c9bafb..ff43b16a2c05 100644 --- a/examples/llm_plus_gnn/multihop_rag/README.md +++ b/examples/llm_plus_gnn/multihop_rag/README.md @@ -1,9 +1,9 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`multihop_download.sh`](./multihop_download.sh) | Downloads all the components of the multihop dataset. | -| [`multihop_preprocess.py`](./multihop_preprocess.py) | Preprocesses the dataset to pair questions/answers with components in the knowledge graph. Contains documentation to describe the process. | -| [`rag_generate_multihop.py`](./rag_generate_multihop.py) | Utilizes the sample remote backend in [`g_retriever_utils`](../g_retriever_utils/) to generate subgraphs for the multihop dataset. | +| Example | Description | +| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| [`multihop_download.sh`](./multihop_download.sh) | Downloads all the components of the multihop dataset. | +| [`multihop_preprocess.py`](./multihop_preprocess.py) | Preprocesses the dataset to pair questions/answers with components in the knowledge graph. Contains documentation to describe the process. | +| [`rag_generate_multihop.py`](./rag_generate_multihop.py) | Utilizes the sample remote backend in [`g_retriever_utils`](../g_retriever_utils/) to generate subgraphs for the multihop dataset. | -NOTE: Performance of GRetriever on this dataset has not been evaluated. \ No newline at end of file +NOTE: Performance of GRetriever on this dataset has not been evaluated. diff --git a/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py b/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py index 4f853406f42b..de93a9e75dd1 100644 --- a/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py +++ b/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py @@ -17,10 +17,12 @@ sys.path.append('..') -from g_retriever_utils.rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 +from g_retriever_utils.rag_backend_utils import \ + create_remote_backend_from_triplets # noqa: E402 from g_retriever_utils.rag_feature_store import \ SentenceTransformerApproxFeatureStore # noqa: E402 -from g_retriever_utils.rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 +from g_retriever_utils.rag_graph_store import \ + NeighborSamplingRAGGraphStore # noqa: E402 # %% parser = argparse.ArgumentParser( diff --git a/examples/llm_plus_gnn/nvtx_examples/README.md b/examples/llm_plus_gnn/nvtx_examples/README.md index 10e0f495a24c..aa4f070d9824 100644 --- a/examples/llm_plus_gnn/nvtx_examples/README.md +++ b/examples/llm_plus_gnn/nvtx_examples/README.md @@ -1,7 +1,7 @@ # Examples for LLM and GNN co-training -| Example | Description | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`nvtx_run.sh`](./nvtx_run.sh) | Runs nsys profiler on a given Python file that contains NVTX calls. | -| [`nvtx_rag_backend_example.py`](./nvtx_rag_backend_example.py) | Example script for nsys profiling a RAG Backend such as that used in [`rag_generate.py`](../g_retriever_utils/rag_generate.py). | -| [`nvtx_webqsp_example.py`](./nvtx_webqsp_example.py) | Example script for nsys profiling the WebQSP dataset. | +| Example | Description | +| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| [`nvtx_run.sh`](./nvtx_run.sh) | Runs nsys profiler on a given Python file that contains NVTX calls. | +| [`nvtx_rag_backend_example.py`](./nvtx_rag_backend_example.py) | Example script for nsys profiling a RAG Backend such as that used in [`rag_generate.py`](../g_retriever_utils/rag_generate.py). | +| [`nvtx_webqsp_example.py`](./nvtx_webqsp_example.py) | Example script for nsys profiling the WebQSP dataset. | diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index fe1cae4d608f..7fb9aa8f399f 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -17,9 +17,12 @@ from torch_geometric.profile.nvtx import nvtxit sys.path.append('..') -from g_retriever_utils.rag_backend_utils import create_remote_backend_from_triplets # noqa: E402 -from g_retriever_utils.rag_feature_store import SentenceTransformerFeatureStore # noqa: E402 -from g_retriever_utils.rag_graph_store import NeighborSamplingRAGGraphStore # noqa: E402 +from g_retriever_utils.rag_backend_utils import \ + create_remote_backend_from_triplets # noqa: E402 +from g_retriever_utils.rag_feature_store import \ + SentenceTransformerFeatureStore # noqa: E402 +from g_retriever_utils.rag_graph_store import \ + NeighborSamplingRAGGraphStore # noqa: E402 # %% # Patch FeatureStore and GraphStore diff --git a/test/nn/models/test_g_retriever.py b/test/nn/models/test_g_retriever.py index 330406d5a801..24a74d1b6f6e 100644 --- a/test/nn/models/test_g_retriever.py +++ b/test/nn/models/test_g_retriever.py @@ -52,6 +52,7 @@ def test_g_retriever() -> None: pred = model.inference(question, x, edge_index, batch, edge_attr) assert len(pred) == 1 + @onlyFullTest @withPackage('transformers', 'sentencepiece', 'accelerate') def test_g_retriever_many_tokens() -> None: @@ -98,4 +99,4 @@ def test_g_retriever_many_tokens() -> None: # Test inference: pred = model.inference(question, x, edge_index, batch, edge_attr) - assert len(pred) == 1 \ No newline at end of file + assert len(pred) == 1 diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 2913d9db948c..b6a8140f83c8 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,5 +1,5 @@ # Code adapted from the G-Retriever paper: https://arxiv.org/abs/2402.07630 -from typing import Any, Dict, List, Tuple, no_type_check +from typing import Any, List, Tuple, no_type_check import numpy as np import torch @@ -213,7 +213,7 @@ def processed_file_names(self) -> List[str]: "pre_transform.pt", "large_graph_indexer", ] - split_file = file_lst.pop(['train','val','test'].index(self.split)) + split_file = file_lst.pop(['train', 'val', 'test'].index(self.split)) file_lst.insert(0, split_file) return file_lst @@ -295,7 +295,7 @@ def _retrieve_subgraphs(self) -> None: self.indexer, (ds['graph'] for ds in self.raw_dataset), pre_transform=preprocess_triplet, verbose=self.verbose) - for index in tqdm(range(len(self.raw_dataset)), + for index in tqdm(range(len(self.raw_dataset)), disable=not self.verbose): data_i = self.raw_dataset[index] graph = next(graph_gen) diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ea5295449fac..ad5e5d281101 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -80,7 +80,8 @@ def __init__( self.projector = torch.nn.Sequential( torch.nn.Linear(mlp_hidden_channels, mlp_hidden_channels), torch.nn.Sigmoid(), - torch.nn.Linear(mlp_hidden_channels, mlp_out_channels * mlp_out_tokens), + torch.nn.Linear(mlp_hidden_channels, + mlp_out_channels * mlp_out_tokens), torch.nn.Unflatten(-1, (mlp_out_tokens, mlp_out_channels)), ).to(self.llm.device) From cd0de7a97e1270accda9650443613a5d96ddc220 Mon Sep 17 00:00:00 2001 From: zaristei Date: Thu, 26 Sep 2024 17:27:47 -0400 Subject: [PATCH 738/752] fix bad inport --- torch_geometric/datasets/web_qsp_dataset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index b6a8140f83c8..c982476795cd 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -6,7 +6,14 @@ from torch import Tensor from tqdm import tqdm -from torch_geometric.data import Data, InMemoryDataset +from torch_geometric.data import ( + Data, + InMemoryDataset, + LargeGraphIndexer, + TripletLike, + get_features_for_triplets_groups, +) + from torch_geometric.nn.nlp import SentenceTransformer From a4964809398eaab2d1f43f0efea11d4ba5279fc2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 01:29:34 +0000 Subject: [PATCH 739/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index c982476795cd..fec68e69c74b 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -13,7 +13,6 @@ TripletLike, get_features_for_triplets_groups, ) - from torch_geometric.nn.nlp import SentenceTransformer From 884ffe569f379e8e5ec35da38a2358c8c26ca5b6 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 00:37:48 -0400 Subject: [PATCH 740/752] bug fix 1 --- torch_geometric/datasets/web_qsp_dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index fec68e69c74b..a40b01d90bac 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,5 +1,5 @@ # Code adapted from the G-Retriever paper: https://arxiv.org/abs/2402.07630 -from typing import Any, List, Tuple, no_type_check +from typing import Any, List, Tuple, no_type_check, Iterator import numpy as np import torch @@ -13,6 +13,8 @@ TripletLike, get_features_for_triplets_groups, ) +from torch_geometric.data.large_graph_indexer import EDGE_RELATION +import os from torch_geometric.nn.nlp import SentenceTransformer From 52944f7abdfe3445bd52f9dab72bd06e8d4562d6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 04:39:06 +0000 Subject: [PATCH 741/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_geometric/datasets/web_qsp_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index a40b01d90bac..7eefa6cf2658 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,5 +1,6 @@ # Code adapted from the G-Retriever paper: https://arxiv.org/abs/2402.07630 -from typing import Any, List, Tuple, no_type_check, Iterator +import os +from typing import Any, Iterator, List, Tuple, no_type_check import numpy as np import torch @@ -14,7 +15,6 @@ get_features_for_triplets_groups, ) from torch_geometric.data.large_graph_indexer import EDGE_RELATION -import os from torch_geometric.nn.nlp import SentenceTransformer From d096fa7be8a6f0b303300436b5eee3bb7f3ff85c Mon Sep 17 00:00:00 2001 From: zaristei Date: Mon, 30 Sep 2024 00:56:23 -0400 Subject: [PATCH 742/752] rename example dir --- examples/{llm => llm_plus_gnn}/README.md | 0 examples/{llm => llm_plus_gnn}/g_retriever.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename examples/{llm => llm_plus_gnn}/README.md (100%) rename examples/{llm => llm_plus_gnn}/g_retriever.py (100%) diff --git a/examples/llm/README.md b/examples/llm_plus_gnn/README.md similarity index 100% rename from examples/llm/README.md rename to examples/llm_plus_gnn/README.md diff --git a/examples/llm/g_retriever.py b/examples/llm_plus_gnn/g_retriever.py similarity index 100% rename from examples/llm/g_retriever.py rename to examples/llm_plus_gnn/g_retriever.py From b89ec88cfa9cc4ebc727912686cac411a0996b32 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 02:02:42 -0400 Subject: [PATCH 743/752] pre-commit --- torch_geometric/datasets/web_qsp_dataset.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 7eefa6cf2658..3b25dd0bf612 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -1,5 +1,6 @@ # Code adapted from the G-Retriever paper: https://arxiv.org/abs/2402.07630 import os +from itertools import chain from typing import Any, Iterator, List, Tuple, no_type_check import numpy as np @@ -176,7 +177,7 @@ def __init__( self.limit = limit self.split = split self.include_pcst = include_pcst - # TODO Confirm why the dependency checks and device setting were removed here + # TODO Confirm why the dependency checks and device setting were removed here # noqa ''' self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") @@ -230,12 +231,15 @@ def _save_raw_data(self) -> None: torch.save(self.split_idxs, self.raw_paths[1]) def _load_raw_data(self) -> None: + import datasets if not hasattr(self, "raw_dataset"): self.raw_dataset = datasets.load_from_disk(self.raw_paths[0]) if not hasattr(self, "split_idxs"): self.split_idxs = torch.load(self.raw_paths[1]) def download(self) -> None: + import datasets + dataset = datasets.load_dataset("rmanluo/RoG-webqsp") self.raw_dataset = datasets.concatenate_datasets( [dataset["train"], dataset["validation"], dataset["test"]]) @@ -332,6 +336,7 @@ def _retrieve_subgraphs(self) -> None: self.save(list_of_graphs, self.processed_paths[0]) def process(self) -> None: + from pandas import DataFrame self._load_raw_data() self.model = SentenceTransformer( 'sentence-transformers/all-roberta-large-v1').to(self.device) From 2b0ccb452b246ff5ec044b8f8d6f4767e1af0467 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 02:02:42 -0400 Subject: [PATCH 744/752] pre-commit --- test/datasets/test_web_qsp_dataset.py | 8 ++++---- torch_geometric/datasets/web_qsp_dataset.py | 13 +++++++++---- torch_geometric/nn/models/g_retriever.py | 6 ++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/test/datasets/test_web_qsp_dataset.py b/test/datasets/test_web_qsp_dataset.py index a4e57e0e48a8..9dbb8218c65a 100644 --- a/test/datasets/test_web_qsp_dataset.py +++ b/test/datasets/test_web_qsp_dataset.py @@ -15,15 +15,15 @@ def test_web_qsp_dataset(): @onlyOnline @onlyFullTest -def test_web_qsp_dataset_limit(): - dataset = WebQSPDataset(limit=100) +def test_web_qsp_dataset_limit(tmp_path): + dataset = WebQSPDataset(root=tmp_path, limit=100) assert len(dataset) == 100 assert str(dataset) == "WebQSPDataset(100)" @onlyOnline @onlyFullTest -def test_web_qsp_dataset_limit_no_pcst(): - dataset = WebQSPDataset(limit=100, include_pcst=False) +def test_web_qsp_dataset_limit_no_pcst(tmp_path): + dataset = WebQSPDataset(root=tmp_path, limit=100, include_pcst=False) assert len(dataset) == 100 assert str(dataset) == "WebQSPDataset(100)" diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 3b25dd0bf612..89f7195d3eba 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -279,13 +279,15 @@ def _build_graph(self) -> None: # Nodes: nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256) # type: ignore + x = self.model.encode(nodes, batch_size=256, + output_device='cpu') # type: ignore self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: edges = self.indexer.get_unique_edge_features( feature_name=EDGE_RELATION) - edge_attr = self.model.encode(edges, batch_size=256) # type: ignore + edge_attr = self.model.encode(edges, batch_size=256, + output_device='cpu') # type: ignore self.indexer.add_edge_feature( new_feature_name="edge_attr", new_feature_vals=edge_attr, @@ -298,7 +300,8 @@ def _build_graph(self) -> None: def _retrieve_subgraphs(self) -> None: print("Encoding questions...") self.questions = [str(ds["question"]) for ds in self.raw_dataset] - q_embs = self.model.encode(self.questions, batch_size=256) + q_embs = self.model.encode(self.questions, batch_size=256, + output_device='cpu') list_of_graphs = [] print("Retrieving subgraphs...") textual_nodes = self.textual_nodes @@ -338,8 +341,10 @@ def _retrieve_subgraphs(self) -> None: def process(self) -> None: from pandas import DataFrame self._load_raw_data() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.model = SentenceTransformer( - 'sentence-transformers/all-roberta-large-v1').to(self.device) + 'sentence-transformers/all-roberta-large-v1').to(device) self.model.eval() if self.force_reload or not os.path.exists(self.processed_paths[-1]): print("Encoding graph...") diff --git a/torch_geometric/nn/models/g_retriever.py b/torch_geometric/nn/models/g_retriever.py index ad5e5d281101..f7529ae721b7 100644 --- a/torch_geometric/nn/models/g_retriever.py +++ b/torch_geometric/nn/models/g_retriever.py @@ -131,6 +131,9 @@ def forward( x = self.projector(x) xs = x.split(1, dim=0) + # Handle case where theres more than one embedding for each sample + xs = [x.squeeze(0) for x in xs] + # Handle questions without node features: batch_unique = batch.unique() batch_size = len(question) @@ -187,6 +190,9 @@ def inference( x = self.projector(x) xs = x.split(1, dim=0) + # Handle case where theres more than one embedding for each sample + xs = [x.squeeze(0) for x in xs] + # Handle questions without node features: batch_unique = batch.unique() batch_size = len(question) From db8085df19719face42ab9316cef915e3d72a558 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 03:33:02 -0400 Subject: [PATCH 745/752] mypy --- torch_geometric/datasets/web_qsp_dataset.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 89f7195d3eba..f0217fbe3bd4 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -278,16 +278,15 @@ def _build_graph(self) -> None: trips, pre_transform=preprocess_triplet) # Nodes: - nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256, - output_device='cpu') # type: ignore + nodes: List[str] = self.indexer.get_unique_node_features() + x = self.model.encode(nodes, batch_size=256, output_device='cpu') self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: - edges = self.indexer.get_unique_edge_features( + edges: List[str] = self.indexer.get_unique_edge_features( feature_name=EDGE_RELATION) edge_attr = self.model.encode(edges, batch_size=256, - output_device='cpu') # type: ignore + output_device='cpu') self.indexer.add_edge_feature( new_feature_name="edge_attr", new_feature_vals=edge_attr, From 1779ce7977d2a1aeac29170d0735d71042336ab3 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 03:38:06 -0400 Subject: [PATCH 746/752] fix import change --- torch_geometric/nn/nlp/llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch_geometric/nn/nlp/llm.py b/torch_geometric/nn/nlp/llm.py index 4a89e352c57a..b58059f8e098 100644 --- a/torch_geometric/nn/nlp/llm.py +++ b/torch_geometric/nn/nlp/llm.py @@ -1,3 +1,4 @@ +import warnings from contextlib import nullcontext from typing import Any, Dict, List, Optional From 7aead2aca7f36da6439646f340d847ff5a0827ee Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 03:45:07 -0400 Subject: [PATCH 747/752] mypy 2 --- torch_geometric/datasets/web_qsp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index f0217fbe3bd4..94d8c0e0fa78 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -284,7 +284,7 @@ def _build_graph(self) -> None: # Edges: edges: List[str] = self.indexer.get_unique_edge_features( - feature_name=EDGE_RELATION) + feature_name=EDGE_RELATION) # type: ignore edge_attr = self.model.encode(edges, batch_size=256, output_device='cpu') self.indexer.add_edge_feature( From d13d2ea3ea5448bdee192d6a934a08c84cfdc09e Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 04:01:35 -0400 Subject: [PATCH 748/752] mypy 3 --- torch_geometric/datasets/web_qsp_dataset.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 94d8c0e0fa78..5ee33f944b9a 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -278,15 +278,17 @@ def _build_graph(self) -> None: trips, pre_transform=preprocess_triplet) # Nodes: - nodes: List[str] = self.indexer.get_unique_node_features() + nodes = self.indexer.get_unique_node_features() x = self.model.encode(nodes, batch_size=256, output_device='cpu') self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: - edges: List[str] = self.indexer.get_unique_edge_features( - feature_name=EDGE_RELATION) # type: ignore - edge_attr = self.model.encode(edges, batch_size=256, - output_device='cpu') + edges = self.indexer.get_unique_edge_features( + feature_name=EDGE_RELATION) + edge_attr = self.model.encode( + edges, # type: ignore + batch_size=256, + output_device='cpu') self.indexer.add_edge_feature( new_feature_name="edge_attr", new_feature_vals=edge_attr, From 04ca150476edbd1f492ab8e8254a2a0d46ae6c21 Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 04:09:31 -0400 Subject: [PATCH 749/752] mypy 4 --- torch_geometric/datasets/web_qsp_dataset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/torch_geometric/datasets/web_qsp_dataset.py b/torch_geometric/datasets/web_qsp_dataset.py index 5ee33f944b9a..fd3caeb99b70 100644 --- a/torch_geometric/datasets/web_qsp_dataset.py +++ b/torch_geometric/datasets/web_qsp_dataset.py @@ -279,7 +279,10 @@ def _build_graph(self) -> None: # Nodes: nodes = self.indexer.get_unique_node_features() - x = self.model.encode(nodes, batch_size=256, output_device='cpu') + x = self.model.encode( + nodes, # type: ignore + batch_size=256, + output_device='cpu') self.indexer.add_node_feature(new_feature_name="x", new_feature_vals=x) # Edges: From 8089b9581c97f604c681256876db3d481893101d Mon Sep 17 00:00:00 2001 From: Zachary Aristei Date: Mon, 30 Sep 2024 04:29:29 -0400 Subject: [PATCH 750/752] fix nvtx examples --- .../llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py | 3 ++- examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py index 7fb9aa8f399f..b30e34b8c7b1 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py @@ -54,7 +54,8 @@ # %% device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = SentenceTransformer().to(device) +model = SentenceTransformer('sentence-transformers/all-roberta-large-v1').to( + device) # %% fs, gs = create_remote_backend_from_triplets( diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py b/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py index 145cc00a410b..a1e9611ee5cc 100644 --- a/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py +++ b/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py @@ -17,6 +17,6 @@ args = parser.parse_args() if args.capture_torch_kernels: with torch.autograd.profiler.emit_nvtx(): - ds = web_qsp_dataset.WebQSPDataset('baseline') + ds = web_qsp_dataset.WebQSPDataset('baseline', limit=10) else: - ds = web_qsp_dataset.WebQSPDataset('baseline') + ds = web_qsp_dataset.WebQSPDataset('baseline', limit=10) From 453d61c87f714d22542e9206324271380870310f Mon Sep 17 00:00:00 2001 From: zaristei Date: Fri, 4 Oct 2024 00:07:12 -0400 Subject: [PATCH 751/752] Rename to path on master branch --- examples/{llm_plus_gnn => llm}/README.md | 0 examples/{llm_plus_gnn => llm}/g_retriever.py | 0 examples/{llm_plus_gnn => llm}/g_retriever_utils/README.md | 0 .../g_retriever_utils/benchmark_model_archs_rag.py | 0 .../{llm_plus_gnn => llm}/g_retriever_utils/rag_backend_utils.py | 0 .../{llm_plus_gnn => llm}/g_retriever_utils/rag_feature_store.py | 0 examples/{llm_plus_gnn => llm}/g_retriever_utils/rag_generate.py | 0 .../{llm_plus_gnn => llm}/g_retriever_utils/rag_graph_store.py | 0 examples/{llm_plus_gnn => llm}/multihop_rag/README.md | 0 examples/{llm_plus_gnn => llm}/multihop_rag/multihop_download.sh | 0 .../{llm_plus_gnn => llm}/multihop_rag/multihop_preprocess.py | 0 .../{llm_plus_gnn => llm}/multihop_rag/rag_generate_multihop.py | 0 examples/{llm_plus_gnn => llm}/nvtx_examples/README.md | 0 .../nvtx_examples/nvtx_rag_backend_example.py | 0 examples/{llm_plus_gnn => llm}/nvtx_examples/nvtx_run.sh | 0 .../{llm_plus_gnn => llm}/nvtx_examples/nvtx_webqsp_example.py | 0 16 files changed, 0 insertions(+), 0 deletions(-) rename examples/{llm_plus_gnn => llm}/README.md (100%) rename examples/{llm_plus_gnn => llm}/g_retriever.py (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/README.md (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/benchmark_model_archs_rag.py (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/rag_backend_utils.py (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/rag_feature_store.py (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/rag_generate.py (100%) rename examples/{llm_plus_gnn => llm}/g_retriever_utils/rag_graph_store.py (100%) rename examples/{llm_plus_gnn => llm}/multihop_rag/README.md (100%) rename examples/{llm_plus_gnn => llm}/multihop_rag/multihop_download.sh (100%) mode change 100755 => 100644 rename examples/{llm_plus_gnn => llm}/multihop_rag/multihop_preprocess.py (100%) rename examples/{llm_plus_gnn => llm}/multihop_rag/rag_generate_multihop.py (100%) rename examples/{llm_plus_gnn => llm}/nvtx_examples/README.md (100%) rename examples/{llm_plus_gnn => llm}/nvtx_examples/nvtx_rag_backend_example.py (100%) rename examples/{llm_plus_gnn => llm}/nvtx_examples/nvtx_run.sh (100%) mode change 100755 => 100644 rename examples/{llm_plus_gnn => llm}/nvtx_examples/nvtx_webqsp_example.py (100%) diff --git a/examples/llm_plus_gnn/README.md b/examples/llm/README.md similarity index 100% rename from examples/llm_plus_gnn/README.md rename to examples/llm/README.md diff --git a/examples/llm_plus_gnn/g_retriever.py b/examples/llm/g_retriever.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever.py rename to examples/llm/g_retriever.py diff --git a/examples/llm_plus_gnn/g_retriever_utils/README.md b/examples/llm/g_retriever_utils/README.md similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/README.md rename to examples/llm/g_retriever_utils/README.md diff --git a/examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py b/examples/llm/g_retriever_utils/benchmark_model_archs_rag.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/benchmark_model_archs_rag.py rename to examples/llm/g_retriever_utils/benchmark_model_archs_rag.py diff --git a/examples/llm_plus_gnn/g_retriever_utils/rag_backend_utils.py b/examples/llm/g_retriever_utils/rag_backend_utils.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/rag_backend_utils.py rename to examples/llm/g_retriever_utils/rag_backend_utils.py diff --git a/examples/llm_plus_gnn/g_retriever_utils/rag_feature_store.py b/examples/llm/g_retriever_utils/rag_feature_store.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/rag_feature_store.py rename to examples/llm/g_retriever_utils/rag_feature_store.py diff --git a/examples/llm_plus_gnn/g_retriever_utils/rag_generate.py b/examples/llm/g_retriever_utils/rag_generate.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/rag_generate.py rename to examples/llm/g_retriever_utils/rag_generate.py diff --git a/examples/llm_plus_gnn/g_retriever_utils/rag_graph_store.py b/examples/llm/g_retriever_utils/rag_graph_store.py similarity index 100% rename from examples/llm_plus_gnn/g_retriever_utils/rag_graph_store.py rename to examples/llm/g_retriever_utils/rag_graph_store.py diff --git a/examples/llm_plus_gnn/multihop_rag/README.md b/examples/llm/multihop_rag/README.md similarity index 100% rename from examples/llm_plus_gnn/multihop_rag/README.md rename to examples/llm/multihop_rag/README.md diff --git a/examples/llm_plus_gnn/multihop_rag/multihop_download.sh b/examples/llm/multihop_rag/multihop_download.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/llm_plus_gnn/multihop_rag/multihop_download.sh rename to examples/llm/multihop_rag/multihop_download.sh diff --git a/examples/llm_plus_gnn/multihop_rag/multihop_preprocess.py b/examples/llm/multihop_rag/multihop_preprocess.py similarity index 100% rename from examples/llm_plus_gnn/multihop_rag/multihop_preprocess.py rename to examples/llm/multihop_rag/multihop_preprocess.py diff --git a/examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py b/examples/llm/multihop_rag/rag_generate_multihop.py similarity index 100% rename from examples/llm_plus_gnn/multihop_rag/rag_generate_multihop.py rename to examples/llm/multihop_rag/rag_generate_multihop.py diff --git a/examples/llm_plus_gnn/nvtx_examples/README.md b/examples/llm/nvtx_examples/README.md similarity index 100% rename from examples/llm_plus_gnn/nvtx_examples/README.md rename to examples/llm/nvtx_examples/README.md diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py b/examples/llm/nvtx_examples/nvtx_rag_backend_example.py similarity index 100% rename from examples/llm_plus_gnn/nvtx_examples/nvtx_rag_backend_example.py rename to examples/llm/nvtx_examples/nvtx_rag_backend_example.py diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_run.sh b/examples/llm/nvtx_examples/nvtx_run.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/llm_plus_gnn/nvtx_examples/nvtx_run.sh rename to examples/llm/nvtx_examples/nvtx_run.sh diff --git a/examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py b/examples/llm/nvtx_examples/nvtx_webqsp_example.py similarity index 100% rename from examples/llm_plus_gnn/nvtx_examples/nvtx_webqsp_example.py rename to examples/llm/nvtx_examples/nvtx_webqsp_example.py From 29cd5bd8e589fbc4c42fc4ab8677739eca0ce303 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 08:09:04 +0000 Subject: [PATCH 752/752] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/llm/README.md | 12 ++++++------ examples/llm/g_retriever.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/llm/README.md b/examples/llm/README.md index e58036fa6ce2..5177e501d610 100644 --- a/examples/llm/README.md +++ b/examples/llm/README.md @@ -1,8 +1,8 @@ # Examples for Co-training LLMs and GNNs -| Example | Description | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training `LLAMA2` with `GAT` for answering questions based on knowledge graph information | -| [`g_retriever_utils/`](./g_retriever_utils/) | Contains multiple scripts for benchmarking GRetriever's architecture and evaluating different retrieval methods. | -| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | -| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | +| Example | Description | +| -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`g_retriever.py`](./g_retriever.py) | Example for Retrieval-Augmented Generation (RAG) w/ GNN+LLM by co-training `LLAMA2` with `GAT` for answering questions based on knowledge graph information | +| [`g_retriever_utils/`](./g_retriever_utils/) | Contains multiple scripts for benchmarking GRetriever's architecture and evaluating different retrieval methods. | +| [`multihop_rag/`](./multihop_rag/) | Contains starter code and an example run for building a Multi-hop dataset using WikiHop5M and 2WikiMultiHopQA | +| [`nvtx_examples/`](./nvtx_examples/) | Contains examples of how to wrap functions using the NVTX profiler for CUDA runtime analysis. | diff --git a/examples/llm/g_retriever.py b/examples/llm/g_retriever.py index 3a9e25b1df12..2844eb11ff3b 100644 --- a/examples/llm/g_retriever.py +++ b/examples/llm/g_retriever.py @@ -11,7 +11,6 @@ import gc import math import multiprocessing as mp -import os.path as osp import re import time from typing import Any, Callable, Dict, List, Type