Merge branch 'master' into pyup-pin-mypy-0.521

Author: SekouD

.travis.yml

@@ -32,3 +32,6 @@ python:
 - 3.4
 - 2.7
 script: tox
+
+after_success:
+

README.rst

@@ -14,11 +14,18 @@ LyricsMaster
         :alt: Documentation Status
 
 .. image:: https://pyup.io/repos/github/SekouD/lyricsmaster/shield.svg
-     :target: https://pyup.io/repos/github/SekouD/lyricsmaster/
-     :alt: Updates
+        :target: https://pyup.io/repos/github/SekouD/lyricsmaster/
+        :alt: Updates
+        :alt: Coverage Status
 
+.. image:: https://codecov.io/gh/SekouD/lyricsmaster/branch/master/graph/badge.svg
+        :target: https://codecov.io/gh/SekouD/lyricsmaster
 
-LyricsMaster is a library for analyzing lyrics using various Machine Learning algorithms. It includes utilities for downloading lyrics from multiple lyrics providers.
+LyricsMaster is a library for downloading lyrics from multiple lyrics providers. The following Lyrics Providers are supported:
+
+- Lyric Wikia
+- The Original Hip-Hop (Rap) Lyrics Archive - OHHLA.com
+- and more to come soon.
 
 
 * Free software: MIT license

lyricsmaster/__init__.py

@@ -4,4 +4,4 @@
 
 __author__ = """SekouD"""
 __email__ = 'sekoud.python@gmail.com'
-__version__ = '0.1.9'
+__version__ = '1.0.2'

lyricsmaster/lyricsmaster.py

@@ -1,3 +1,65 @@
 # -*- coding: utf-8 -*-
 
 """Main module."""
+
+class Song:
+    def __init__(self, title, album, author, lyrics):
+        self.title = title
+        self.album = album
+        self.author = author
+        self.lyrics = lyrics
+
+    def __repr__(self):
+        return self.__class__.__name__ + " Object: " + self.title
+
+
+class Album:
+    def __init__(self, title, author, songs):
+        self.idx = 0
+        self.title = title
+        self.author = author
+        self.songs = songs
+
+    def __repr__(self):
+        return self.__class__.__name__ + " Object: " + self.title
+
+    def __len__(self):
+        return len(self.songs)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        self.idx += 1
+        try:
+            return self.songs[self.idx - 1]
+        except IndexError:
+            self.idx = 0
+            raise StopIteration
+
+    next = __next__
+
+class Discography:
+    def __init__(self, author, albums):
+        self.idx = 0
+        self.author = author
+        self.albums = albums
+
+    def __repr__(self):
+        return self.__class__.__name__ + " Object: " + self.author
+
+    def __len__(self):
+        return len(self.albums)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        self.idx += 1
+        try:
+            return self.albums[self.idx - 1]
+        except IndexError:
+            self.idx = 0
+            raise StopIteration
+
+    next = __next__

lyricsmaster/lyricsprovider.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+"""Lyrics Providers."""
+from .lyricsmaster import Song, Album, Discography
+import requests
+import bs4
+from bs4 import BeautifulSoup
+
+
+class LyricsProvider:
+    def get_page(self, url):
+        try:
+            req = requests.get(url)
+        except:
+            req = None
+            print('Unable to download url ' + url)
+        return req
+
+    def get_lyrics(self, author):
+        pass
+
+    def get_artist_page(self, author):
+        pass
+
+    def get_album_page(self, author, album):
+        pass
+
+    def get_lyrics_page(self, author, album, title):
+        pass
+
+    def extract_lyrics(self, song):
+        pass
+
+
+class LyricWiki(LyricsProvider):
+    base_url = 'http://lyrics.wikia.com'
+
+    def clean_string(self, text):
+        text = text.replace('#', 'Number_').replace('[', '(').replace(']', ')').replace('{', '(').replace('}', ')')
+        text = text.replace(' ', '_')
+        return text
+
+    def get_artist_page(self, author):
+        author = self.clean_string(author)
+        url = self.base_url + '/wiki/' + author
+        artist_page = BeautifulSoup(self.get_page(url).text, 'lxml')
+        return artist_page
+
+    def get_album_page(self, author, album):
+        author = self.clean_string(author)
+        album = self.clean_string(album)
+        url = self.base_url + '/wiki/' + author + ':' + album
+        album_page = BeautifulSoup(self.get_page(url).text, 'lxml')
+        return album_page
+
+    def get_lyrics_page(self, url):
+        lyrics_page = BeautifulSoup(self.get_page(url).text, 'lxml')
+        return lyrics_page
+
+    def extract_lyrics(self, song):
+        lyric_box = song.find("div", {'class': 'lyricbox'})
+        lyrics = lyric_box.text
+        return lyrics
+
+    def get_songs(self, album):
+        parent_node = album.parent
+        while parent_node.name != 'ol':
+            parent_node = parent_node.next_sibling
+        song_links = parent_node.find_all('li')
+        return song_links
+
+    def create_song(self, link, author, album_title):
+        link = link.find('a')
+        song_title = link.attrs['title']
+        if '(page does not exist' in song_title:
+            return None
+        lyrics_page = self.get_lyrics_page(self.base_url + link.attrs['href'])
+        lyrics = self.extract_lyrics(lyrics_page)
+        song = Song(song_title, album_title, author, lyrics)
+        return song
+
+    def get_lyrics(self, author):
+        artist_page = self.get_artist_page(author)
+        albums = [tag for tag in artist_page.find_all("span", {'class': 'mw-headline'}) if
+                  tag.attrs['id'] not in ('Additional_information', 'External_links')]
+        album_objects = []
+        for elmt in albums:
+            album_title = elmt.text
+            song_links = self.get_songs(elmt)
+            songs = [self.create_song(link, author, album_title) for link in song_links]
+            album = Album(album_title, author, songs)
+            album_objects.append(album)
+        discography = Discography(author, album_objects)
+        return discography
+
+
+
+
+
+
+class RapProvider(LyricsProvider):
+    base_url = "http://ohhla.com/all{0}.html"
+    pass
+
+if __name__ == "__main__":
+    test = LyricWiki()
+    artist = test.get_artist_page('2Pac')
+    album = test.get_album_page('2Pac', 'Me Against The World (1995)')
+    lyrics_page = test.get_lyrics_page('http://lyrics.wikia.com/wiki/2Pac:Young_Black_Male')
+    lyrics = test.extract_lyrics(lyrics_page)
+    test_wikia = test.get_lyrics('Reggie Watts')
+    pass

requirements.txt

@@ -1,5 +1,2 @@
-scikit-learn>=0.18
-git+https://github.com/clips/pattern.git@development
-pandas==0.20.3
-h5py>=2.7
-matplotlib>=2
+beautifulsoup4==4.6.0
+requests

requirements_dev.txt

@@ -9,3 +9,8 @@ Sphinx==1.6.3
 cryptography==2.0.3
 PyYAML==3.12
 mypy==0.521
+pytest==3.2.2
+hypothesis==3.28.1
+beautifulsoup4==4.6.0
+requests
+

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.9
+current_version = 1.0.2
 commit = True
 tag = True
 

setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name='lyricsmaster',
-    version='0.1.9',
+    version='1.0.2',
     description="LyricsMaster is a library for analyzing lyrics using various Machine Learning algorithms. It includes utilities for downloading lyrics from multiple lyrics providers.",
     long_description=readme + '\n\n' + history,
     author="SekouD",

tests/test_lyricsmaster.py

@@ -4,31 +4,129 @@
 """Tests for `lyricsmaster` package."""
 
 
-import unittest
+from collections import Iterable
+import pytest
 from click.testing import CliRunner
 
+from bs4 import BeautifulSoup, Tag
+
 from lyricsmaster import lyricsmaster
 from lyricsmaster import cli
+from lyricsmaster import lyricsprovider
+
+try:
+  basestring
+except NameError:
+  basestring = str
+
+@pytest.fixture(scope="module")
+def songs():
+    songs = [lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None),
+             lyricsmaster.Song('Ragged and dirty', 'Bad news is coming', 'Luther Alison', None),
+             lyricsmaster.Song('Red rooster', 'Bad news is coming', 'Luther Alison', None),
+             lyricsmaster.Song('Life is bitch', 'Bad news is coming', 'Luther Alison', None)]
+    return songs
+
+class TestSongs:
+    """Tests for Song Class."""
+    song = lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None)
+
+    def test_song(self):
+        assert self.song.__repr__() == 'Song Object: Bad Love'
+
+
+class TestAlbums:
+    """Tests for Album Class."""
+
+    songs = songs()
+    album = lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)
+
+    def test_album(self):
+        assert self.album.idx == 0
+        assert self.album.title == 'Bad news is coming'
+        assert self.album.author == 'Luther Alison'
+        assert self.album.__repr__() == 'Album Object: Bad news is coming'
+
+    def test_album_isiter(self):
+        assert len(self.album) == 4
+        assert [elmt for elmt in self.album] == self.songs
+
+
+class TestDiscography:
+    """Tests for Album Class."""
+
+    albums = [lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs),
+                       lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)]
+    discography = lyricsmaster.Discography('Luther Allison', albums)
+
+    def test_discography(self):
+        assert  self.discography.__repr__() == 'Discography Object: Luther Allison'
+
+    def test_discography_isiter(self):
+        assert len(self.discography) == 2
+        assert [elmt for elmt in self.discography] == self.albums
+
+class TestLyricWiki:
+    provider = lyricsprovider.LyricWiki()
+    author = 'Reggie Watts'
+
+    def test_clean_string(self):
+        assert self.provider.clean_string('Reggie Watts {(#5)}') == 'Reggie_Watts_((Number_5))'
+
+    def test_get_artist_page(self):
+        page = self.provider.get_artist_page(self.author)
+        assert isinstance(page, BeautifulSoup)
+
+    def test_get_album_page(self):
+        page = self.provider.get_album_page('Reggie Watts', 'Simplified (2004)')
+        assert isinstance(page, BeautifulSoup)
+
+    def test_get_lyrics_page(self):
+        page = self.provider.get_lyrics_page('http://lyrics.wikia.com/wiki/Reggie_Watts:Your_Name')
+        assert isinstance(page, BeautifulSoup)
+
+    def test_extract_lyrics(self):
+        page = self.provider.get_lyrics_page('http://lyrics.wikia.com/wiki/Reggie_Watts:Your_Name')
+        lyrics = self.provider.extract_lyrics(page)
+        assert isinstance(lyrics, basestring)
+        assert 'I recall the day' in lyrics
+        assert "And I hope you'll stay." in lyrics
 
+    def test_get_songs(self):
+        author_page = self.provider.get_artist_page(self.author)
+        album = [tag for tag in author_page.find_all("span", {'class': 'mw-headline'}) if
+                  tag.attrs['id'] not in ('Additional_information', 'External_links')][0]
+        song_links = self.provider.get_songs(album)
+        for link in song_links:
+            assert isinstance(link, Tag)
 
-class TestLyricsmaster(unittest.TestCase):
-    """Tests for `lyricsmaster` package."""
+    def test_create_song(self):
+        author_page = self.provider.get_artist_page(self.author)
+        album = [tag for tag in author_page.find_all("span", {'class': 'mw-headline'}) if
+                 tag.attrs['id'] not in ('Additional_information', 'External_links')][0]
+        song_links = self.provider.get_songs(album)
+        fail_song = self.provider.create_song(song_links[0], self.author, "Simplified (2004)")
+        assert fail_song is None
+        good_song = self.provider.create_song(song_links[9], self.author, "Simplified (2004)")
+        assert isinstance(good_song, lyricsmaster.Song)
+        assert good_song.title == 'Reggie Watts:Your Name'
+        assert good_song.album == "Simplified (2004)"
+        assert good_song.author == self.author
+        assert 'I recall the day' in good_song.lyrics
+        assert "And I hope you'll stay." in good_song.lyrics
 
-    def setUp(self):
-        """Set up test fixtures, if any."""
+    def test_get_lyrics(self):
+        discography = self.provider.get_lyrics(self.author)
+        assert isinstance(discography, lyricsmaster.Discography)
 
-    def tearDown(self):
-        """Tear down test fixtures, if any."""
 
-    def test_000_something(self):
-        """Test something."""
 
-    def test_command_line_interface(self):
-        """Test the CLI."""
-        runner = CliRunner()
-        result = runner.invoke(cli.main)
-        assert result.exit_code == 0
-        assert 'lyricsmaster.cli.main' in result.output
-        help_result = runner.invoke(cli.main, ['--help'])
-        assert help_result.exit_code == 0
-        assert '--help  Show this message and exit.' in help_result.output
+def test_command_line_interface():
+    """Test the CLI."""
+    runner = CliRunner()
+    result = runner.invoke(cli.main)
+    assert result.exit_code == 0
+    assert 'lyricsmaster.cli.main' in result.output
+    help_result = runner.invoke(cli.main, ['--help'])
+    assert help_result.exit_code == 0
+    assert '--help  Show this message and exit.' in help_result.output