CF considering neighbor

유사도가 높은 사람만 이웃으로 선정해서 이웃의 크기를 줄임
즉, 유사도가 낮은 사람의 rating이 반영되는 것을 막음

이웃을 정하는 기준

1) kNN
2) 특정 유사도 값 이상만 사용 (thresholding)

thresholding은 특정 값 이상의 data가 없을 경우 정상 동작 하지 않기에, kNN을 많이 사용함
 

read data

 

import numpy as np
import pandas as pd

u_cols = ['user_id', 'age', 'sex', 'occupation', 'zip_code']
users = pd.read_csv('u.user', sep='|', names=u_cols, encoding='latin-1')
i_cols = ['movie_id', 'title', 'release date', 'video release date', 'IMDB URL', 'unknown', 
          'Action', 'Adventure', 'Animation', 'Children\'s', 'Comedy', 'Crime', 'Documentary', 
          'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 
          'Thriller', 'War', 'Western']
movies = pd.read_csv('u.item', sep='|', names=i_cols, encoding='latin-1')
r_cols = ['user_id', 'movie_id', 'rating', 'timestamp']
ratings = pd.read_csv('u.data', sep='\t', names=r_cols, encoding='latin-1')

 

cleansing

 

ratings = ratings.drop('timestamp', axis=1)
movies = movies[['movie_id', 'title']]

 

split data set

 

from sklearn.model_selection import train_test_split
from sklearn.metrics.pairwise import cosine_similarity

 

x = ratings.copy()
y = ratings['user_id']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, stratify=y)

 

define score/RMSE

 

# 정확도(RMSE)를 계산하는 함수 
def RMSE(y_true, y_pred):
    return np.sqrt(np.mean((np.array(y_true) - np.array(y_pred))**2))

# 모델별 RMSE를 계산하는 함수 
def score(model, neighbor_size=0):
    id_pairs = zip(x_test['user_id'], x_test['movie_id'])
    y_pred = np.array([model(user, movie, neighbor_size) for (user, movie) in id_pairs])
    y_true = np.array(x_test['rating'])
    return RMSE(y_true, y_pred)

 

create rating matrix

 

rating_matrix = x_train.pivot(index='user_id', columns='movie_id', values='rating')
matrix_dummy = rating_matrix.copy().fillna(0)
user_similarity = cosine_similarity(matrix_dummy, matrix_dummy)
user_similarity = pd.DataFrame(user_similarity, index=rating_matrix.index, columns=rating_matrix.index)

 

define CF with kNN

 

def cf_knn(user_id, movie_id, neighbor_size=0):
    if movie_id not in rating_matrix:
        return 3.0

    sim_scores = user_similarity[user_id].copy()   # get user similarity vector
    movie_ratings = rating_matrix[movie_id].copy()  # get movie's rating vector

    none_rating_idx = movie_ratings[movie_ratings.isnull()].index  # remove not rated user rows
    movie_ratings = movie_ratings.drop(none_rating_idx)

    sim_scores = sim_scores.drop(none_rating_idx)   # remove not rated user rows in sim vector

    if neighbor_size == 0:          
        # get mean from all users
        return np.dot(sim_scores, movie_ratings) / sim_scores.sum()

    if len(sim_scores) > 1: 
        # 지정된 neighbor size 값과 해당 영화를 평가한 총사용자 수 중 작은 것으로 결정
        neighbor_size = min(neighbor_size, len(sim_scores))
        # array로 바꾸기 (argsort를 사용하기 위함)
        sim_scores = np.array(sim_scores)
        movie_ratings = np.array(movie_ratings)

        user_idx = np.argsort(sim_scores)   # sort and get sorted index
        sim_scores = sim_scores[user_idx][-neighbor_size:]  # sim scores as much as user_idx size

        movie_ratings = movie_ratings[user_idx][-neighbor_size:]

        mean_rating = np.dot(sim_scores, movie_ratings) / sim_scores.sum()
    else:
        mean_rating = 3.0

    return mean_rating    

 

score(cf_knn, neighbor_size=30)

    1.0069864963443693

 

get N-best

def recom_movie(user_id, n_items, neighbor_size=30):
    user_movie = rating_matrix.loc[user_id].copy()
    for movie in rating_matrix:        
        user_movie.loc[movie] = cf_knn(user_id, movie, neighbor_size)

    movie_sort = user_movie.sort_values(ascending=False)[:n_items]
    recom_movies = movies.loc[movie_sort.index]
    recommendations = recom_movies['title']
    return recommendations

movie_id
119                      Striptease (1996)
1293      Ayn Rand: A Sense of Life (1997)
1189               That Old Feeling (1997)
1467                      Cure, The (1995)
64      What's Eating Gilbert Grape (1993)
Name: title, dtype: object