Last.fm Music Recommender

import time
import re
from bs4 import BeautifulSoup
import urllib.request
import numpy as np
import progressbar
import pylast
import os
import operator
import scipy.sparse as sp
import matplotlib.pyplot as plt
from collections import OrderedDict

API_KEY = "389968e3566746e89cbb4750271c3aa1"
API_SECRET = "a96e6333143c6f686fd045a41acf1f4e"
PATH_USERNAMES = "dataset/usernames.npy"
PATH_SONGS = "dataset/songs.npy"
PATH_RATINGS = "dataset/ratings.npy"
PATH_MATRIX = "dataset/matrix.npz"
PATH_RATING_DICT = "dataset/rating_dict.npy"
PATH_MODEL = "model/ae.model"

def get_username_from_url(request):
    res = set()
    with urllib.request.urlopen(request) as urlObject:
        html = urlObject.read()
        soup = BeautifulSoup(html, "html.parser")
        results = soup.findAll('li', { "class" : "b_algo" })
        for result in results:
            result = str(result.find('a').get('href'))
            username = re.search(r'https://[^.]+.last.fm/user/([^/]+)', result)
            if username is not None:
                res.add(username.group(1))
            else:
                print('failed to fetch username in', result)
    return res

def bing_search():
    print('crawling usernames from Bing')
    url = "https://www.bing.com/search?q=site%3alast.fm%2fuser%2f&qs=ds&first={0}&FORM=PERE"
    headers = {'User-Agent':'Chro/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0'}
    res = set()
    start = 1
    address = url.format(start)
    request = urllib.request.Request(address, None, headers)
    res = res.union(get_username_from_url(request))
    start = 15
    total = 1000
    with progressbar.ProgressBar(max_value=total) as bar:
        while start < total:
            address = url.format(start)
            request = urllib.request.Request(address, None, headers)     
            res = res.union(get_username_from_url(request))
            start += 10
            bar.update(min(start, total))
            time.sleep(1)
    print('crawled %d usernames from bing search'%len(res))
    return res

network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET)

def crawl_user_friends(usernames, depth=1, max_friends=0):
    usernames = usernames[:]
    for i in range(depth):
        print('fetching list of usernames with a depth of %d'%depth)
        assert len(usernames) == i+1
        friends = set()
        # traverse each user in last depth
        with progressbar.ProgressBar(max_value=len(usernames[i])) as bar:
            for j, username in enumerate(usernames[i]):
                user = network.get_user(username)
                try:
                    n = 0
                    for friend in user.get_friends():
                        recorded = False
                        for prev in usernames: # check if friend is already recorded in previous depths
                            if friend.get_name() in prev:
                                recorded = True
                        if not recorded:
                            friends.add(friend.get_name())
                            n += 1
                        if max_friends and n >= max_friends:
                            break
                except:
                    pass
                finally:
                    bar.update(j)
        usernames.append(friends)
    return usernames

def get_usernames():
    # TODO refer to cache funtions in previous hw
    path = os.path.join(os.getcwd(), PATH_USERNAMES)
    if os.path.isfile(path):
        return np.load(path, allow_pickle=True)
    dataset = crawl_user_friends([bing_search()], depth=1, max_friends=20)
    dataset = np.array(list(set.union(*dataset)), dtype=object)
    np.save(path, dataset)
    return dataset

USERNAMES = get_usernames()

import xml.dom
def _collect_nodes(limit, sender, method_name, cacheable, params=None):
    """
    Returns a sequence of dom.Node objects about as close to limit as possible
    """

    if not params:
        params = sender._get_params()

    nodes = []
    page = 1
    end_of_pages = False

    while not end_of_pages:
        params["page"] = str(page)

        tries = 1
        while True:
            try:
                doc = sender._request(method_name, cacheable, params)
                break  # success
            except Exception as e:
                if tries >= 3:
                    raise e
                # Wait and try again
                time.sleep(1)
                tries += 1

        doc = pylast.cleanup_nodes(doc)

        # break if there are no child nodes
        if not doc.documentElement.childNodes:
            break
        main = doc.documentElement.childNodes[0]

        if main.hasAttribute("totalPages"):
            total_pages = pylast._number(main.getAttribute("totalPages"))
        elif main.hasAttribute("totalpages"):
            total_pages = pylast._number(main.getAttribute("totalpages"))
        else:
            raise Exception("No total pages attribute")

        for node in main.childNodes:
            if not node.nodeType == xml.dom.Node.TEXT_NODE:
                nodes.append(node)

        if page >= total_pages:
            end_of_pages = True

        page += 1

    return nodes

# overide pylast's method
pylast._collect_nodes = _collect_nodes

def get_dictionary(usernames=USERNAMES):
    
    # check if there exists rating matrix
    path_rating_dict = os.path.join(os.getcwd(), PATH_RATING_DICT)
    if os.path.isfile(path_rating_dict):
        return np.load(path_rating_dict, allow_pickle=True).item()
    
    d = {} # {username: {(song, artist): playcount}}
    with progressbar.ProgressBar(max_value=len(usernames)) as bar:
        for user_index, username in enumerate(usernames):
            try:
                user = network.get_user(username)
                tracks = user.get_top_tracks(period="1month", limit=1000, cacheable=True) # todo test cacheable
                if username not in d and len(tracks):
                    d[username] = {}
                for item in tracks:
                    track, play_count = item
                    key =(track.get_artist().get_name(), track.get_name())
                    d[username][key] = play_count
            except Exception as e:
                pass
            finally:
#                 with open(path_rating_dict, 'w') as f:
                np.save(path_rating_dict, d)
    return d

def normalize_ratings(play_count):
    if not len(play_count):
        return play_count
    play_count = np.log(np.array(play_count, dtype=np.float))
#     play_count = np.sqrt(np.log(np.array(play_count, dtype=np.float)))
    max_count = np.max(play_count)
    if max_count > 0:
        play_count /= max_count
    play_count = play_count*0.9 + 0.1
    return play_count.tolist()

def purge_rating_dict(user_play_dict, min_songs, min_users):
    """
    1. remove users with only a few played songs (<min_songs)
    2. remove songs played by only a few users (<min_users)
    3. normalize playcounts
    4. return user_index_list, song_index_list, user_list, song_list, rating_list
    """
    # List for build up Matrix
    user_indices = []
    song_indices = []

    rating_list = []
    user_list = []
    song_list = []

    # remove users with only a few played songs (<min_songs)
    remove_user_list = []
    print('user num:', len(user_play_dict), end='')
    for user in user_play_dict:
        if len(user_play_dict[user]) < min_songs:
            remove_user_list.append(user)
            
    for key in remove_user_list:
        if key in user_play_dict:
            del user_play_dict[key]
            
    print('->', len(user_play_dict), end='n')
    
    # remove songs played by only a few users (<min_users)
    song_dict = {}
    remove_songs = set()
    for keys, values in user_play_dict.items():
        for song in values:
            song_dict[song] = song_dict.get(song, 0) + 1
    print('song num :', len(song_dict), end='')
    for key in song_dict:
        if song_dict[key] < min_users:
            remove_songs.add(key)
            
    for key in remove_songs:
        if key in song_dict:
            del song_dict[key]
    print('->', len(song_dict), end='n')
    
    d = {}
    for name in user_play_dict.keys():
        ratings = []
        _remove_songs = []
        _remove_users = []
        for k, v in user_play_dict[name].items():
            if k in remove_songs:
                _remove_songs.append(k)
                continue
            if k not in d:
                song_indices.append(k)
                d[k] = len(d)
            user_list.append(len(user_indices))
            song_list.append(d[k])
            ratings.append(v)
        if len(ratings):
            rating_list.extend(normalize_ratings(ratings))
            user_indices.append(name)
        else:
            _remove_users.append(name)
        for s in _remove_songs:
            if s in user_play_dict[name]:
                del user_play_dict[name][s]
    for u in _remove_users:
        if u in user_play_dict:
            del user_play_dict[u]
    
    return rating_list, user_list, song_list, user_indices, song_indices

def processVisual(dt):
    song_dict = {}
    user_dict = {}
    remove_songs = set()
    for key, values in dt.items():
        user_dict[key] = len(values)
        for song in values:
            song_dict[song] = song_dict.get(song, 0) + 1
        
    return song_dict, user_dict

song_dict, user_dict = processVisual(user_play_dict)

from sklearn.model_selection import train_test_split
def split_dataset(data, row, col, val_size=0.2, test_size=0.1, random_state=0xDECAFE):

    train_userid, test_userid, train_songid, test_songid, train_rating, test_rating = train_test_split(row, col, data, test_size=0.1, random_state=random_state)
    train_userid, val_userid, train_songid, val_songid, train_rating, val_rating = train_test_split(train_userid, train_songid, train_rating, test_size=0.2, random_state=random_state)
    
    # Split matrix
    matrix_train = sp.coo_matrix((train_rating, (train_userid, train_songid)), shape=(len(user_indices), len(song_indices)))
    matrix_val = sp.coo_matrix((val_rating, (val_userid, val_songid)), shape=(len(user_indices), len(song_indices)))
    matrix_test = sp.coo_matrix((test_rating, (test_userid,
                                               test_songid)), shape=(len(user_indices), len(song_indices)))
    return matrix_train, matrix_val, matrix_test

matrix_train, matrix_val, matrix_test = split_dataset(rating_list, user_list, song_list)

# cite: error computation taken from 688 hw4_cf
import scipy.linalg as la
import time

def error(X, U, V):
    """ Compute the mean error of the observed ratings in X and their estimated values. 

        args: 
            X : np.array[num_users, num_movies] -- the ratings matrix
            U : np.array[num_users, num_features] -- a matrix of features for each user
            V : np.array[num_movies,num_features] -- a matrix of features for each movie

        return: float -- the mean squared error between non-zero entries of X and the ratings
            predicted by U and V; as this is an error and not a loss function, you do not need to include the
            regularizing terms.
        """
    return np.sum(np.square(np.multiply(np.sign(X), (X - np.dot(U, V)))))/np.sum(X!=0)

svd_u, svd_s, svd_vt = sp.linalg.svds(matrix_train, k=300)

import os
import time
import math
import torch
import numpy as np
import torch.nn as nn
import scipy.sparse as sp
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader

cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")

class RatingDataLoader(DataLoader):

    def __init__(self, dataset, batch_size, shuffle=True):
        
        # super(RatingDataLoader, self).__init__(dataset=dataset, batch_size=batch_size, shuffle=shuffle) 
        self.dataset = dataset
        self.batch_size = batch_size
        
        # dict{usrid: matrix_row_indices}
        self.usr_dict = self.get_usr_dict(dataset.row)
        self.usr_list = sorted(self.usr_dict.keys())
        self.num_row = dataset.get_shape()[0]
        self.num_col = dataset.get_shape()[1]
        
        self.shuffle = shuffle
        self.length = math.ceil(len(self.usr_list)/self.batch_size)
        
    def __len__(self):
        return self.length

    def __iter__(self):
        usr_list = self.usr_list.copy()
        
        # random
        if self.shuffle:
            np.random.shuffle(usr_list)
        
        while len(usr_list) > 0:
            indices_row = []
            indices_col = []
            for i in range(self.batch_size):
                if len(usr_list) == 0:
                    break
                idx = self.usr_dict[usr_list.pop()]
                indices_col.extend(idx)
                indices_row.extend([i] * len(idx))
                
            row = np.array(indices_row, dtype=np.int32)
            col = self.dataset.col[indices_col]
            data = torch.FloatTensor(self.dataset.data[indices_col])
            row_col = torch.LongTensor(np.stack((row, col)))
            
            yield torch.sparse.FloatTensor(row_col, data, torch.Size([indices_row[-1]+1, self.num_col])).to_dense()
    
    def get_usr_dict(self, row_list):
        d = {}
        for i, u in enumerate(row_list):
            if u in d:
                d[u].append(i)
            else:
                d[u] = [i]
        return d

class Autoencoder(nn.Module):
    """
    symetrical encoder-decoder layers
    """
    
    def __init__(self, layer_sizes, dropout_prob):
        super().__init__()
        self.layer_sizes = layer_sizes
        self.dropout_prob = dropout_prob
        self.encoder = nn.Sequential(*self.getLayers("encoder"))
        self.decoder = nn.Sequential(*self.getLayers("decoder"))
        
    def getLayers(self, type):
        layers = []
        if type == "encoder":
            layers.append(nn.Dropout(self.dropout_prob))
            _layer_sizes = self.layer_sizes
        elif type == "decoder":
            _layer_sizes = self.layer_sizes[: : -1]
        for i in range(len(_layer_sizes)-1):
            layers.append(nn.Linear(_layer_sizes[i], _layer_sizes[i+1]))
            layers.append(nn.Tanh())
        return layers
        
    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x

def train_epoch(model, train_loader, criterion, optimizer):
    model.train()  # set to training mode
    model.to(device)
    running_loss = 0.0
    start_time = time.time()
    for batch_idx, data in enumerate(train_loader):
        optimizer.zero_grad()  # clear gradient for each batch
        data = data.to(device)
        
        # forward
        train_out = model(data)
        loss = criterion(train_out, data)
        running_loss += loss.item()
        
        # backward
        loss.backward()
        optimizer.step()
    end_time = time.time()
    running_loss /= len(train_loader)
    print("Reconstruction Loss: %f Time: %fs" % (running_loss, end_time - start_time))
    return running_loss

def eval_epoch(model, eval_loader, criterion):
    with torch.no_grad():
        model.eval()
        model.to(device)
        running_loss = 0.0
        total_pred = 0
        correct_pred = 0
        for batch_idx, data in enumerate(eval_loader):
            data = data.to(device)
            eval_out = model(data)
            total_pred = data.size(0)
            correct_pred += (eval_out == data).sum().item()
            loss = criterion(eval_out, data).detach()
            running_loss += loss.item()
        running_loss /= len(eval_loader)
        print("Validation Loss: %f" % running_loss)
        return running_loss

class zero_masked_MSELoss(nn.Module):
    def __init__(self):
        super().__init__()
    
    def forward(self, input, target):
        diff = input - target
        mask = torch.where(target != 0, torch.ones_like(target), target)
        return torch.sum(diff ** 2 * mask) / torch.sum(mask)

def train_autoencoder(model, path):
    
    # load data
    print("loading data")

    batch_size = 256
#     matrix_combined = (matrix_train + matrix_val).tocoo()
    train_loader = RatingDataLoader(matrix_train, batch_size, shuffle=True)
    test_loader = RatingDataLoader(matrix_val, batch_size, shuffle=False)
    
    criterion = zero_masked_MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # training loop
    n_epochs = 250
    print("training autoencoder for %d epochs:" % n_epochs)
    train_loss = []
    val_loss = []
    with progressbar.ProgressBar(max_value=n_epochs) as bar:
        for i in range(n_epochs):
#             print("Epoch: %d/%d"%((i+1),n_epochs))
            t_loss = train_epoch(model, train_loader, criterion, optimizer)
            v_loss = eval_epoch(model, test_loader, criterion)
            train_loss.append(t_loss)
            val_loss.append(v_loss)
            bar.update(i)
    print("train_loss=%ftval_loss=%f"%(train_loss[-1], val_loss[-1]))
    print("done")
    torch.save({
            'model_state_dict': model.state_dict(),
            'train_loss': train_loss,
            'val_loss': val_loss,
            }, path)
    return train_loss, val_loss

def plot_loss(train_loss, val_loss):

    train_loss = np.array(train_loss)
    val_loss = np.array(val_loss)

    fig, ax = plt.subplots()
    ax.set(xlabel='EPOCH', ylabel='LOSS',title='LOSS')
    ax.grid()
    
    ax.plot(np.arange(len(train_loss))+1, train_loss, label='train loss')
    ax.plot(np.arange(len(val_loss))+1, val_loss, label='val loss')
    plt.legend(title='')
    plt.show()
    

def get_recommender_AE():
    recommender_AE = Autoencoder([matrix_train.get_shape()[1], 1000, 1000, 400], dropout_prob=0.5)
    print(recommender_AE)
    path_model = os.path.join(os.getcwd(), PATH_MODEL)
    if os.path.isfile(path_model):
        checkpoint = torch.load(path_model)
        recommender_AE.load_state_dict(checkpoint['model_state_dict'])
        train_loss = checkpoint['train_loss']
        val_loss = checkpoint['val_loss']
    else:
        train_loss, val_loss = train_autoencoder(recommender_AE, path_model)
    plot_loss(train_loss, val_loss)
    return recommender_AE.to(device)

# loading test data
test_data = matrix_test.toarray()
pred_MF = (svd_u@np.diag(svd_s))@svd_vt

train_data = matrix_train.toarray()
tensor = torch.from_numpy(train_data).float()
tensor = tensor.to(device)

with torch.no_grad():
    recommender_AE.eval()
    pred_AE = recommender_AE(tensor)

pred_AE = pred_AE.to("cpu").detach().numpy()

"""
returns an MSE between the input matrix and the target matrix
The mask is defined by a tuple of rating threadsholds
"""
def threshold_MSE(input, target, threshold):
    diff = input - target
    start, end = threshold
    mask = np.where((target>start)&(target<end),1,0)
    return np.sum(diff ** 2 * mask) / np.sum(mask)

thresholds = [(0.1, 0.2), (0.2, 0.4), (0.4, 0.6), (0.6, 0.8), (0.8, 1.0)]
threshold_MSE_MF = []
threshold_MSE_AE = []

for t in thresholds:
    threshold_MSE_MF.append(threshold_MSE(pred_MF, test_data, t))
    threshold_MSE_AE.append(threshold_MSE(pred_AE, test_data, t))