librefm-commits
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Librefm-commits] [1218] merge scripts from trunk


From: Clint Adams
Subject: [Librefm-commits] [1218] merge scripts from trunk
Date: Wed, 06 May 2009 00:25:03 +0000

Revision: 1218
          http://svn.sv.gnu.org/viewvc/?view=rev&root=librefm&revision=1218
Author:   clint
Date:     2009-05-06 00:25:02 +0000 (Wed, 06 May 2009)
Log Message:
-----------
merge scripts from trunk

Modified Paths:
--------------
    branches/stable/scripts/import-jamendo.py
    branches/stable/scripts/scrobbles-revisionism
    branches/stable/scripts/users_dump.sh

Added Paths:
-----------
    branches/stable/scripts/countrycodes.pl
    branches/stable/scripts/gobble.py
    branches/stable/scripts/most_recent.py
    branches/stable/scripts/offline-submit.py
    branches/stable/scripts/setstids

Removed Paths:
-------------
    branches/stable/scripts/users.sql

Added: branches/stable/scripts/countrycodes.pl
===================================================================
--- branches/stable/scripts/countrycodes.pl                             (rev 0)
+++ branches/stable/scripts/countrycodes.pl     2009-05-06 00:25:02 UTC (rev 
1218)
@@ -0,0 +1,54 @@
+#!/usr/bin/perl
+
+use CGI qw(escape);
+use Encode qw(_utf8_on);
+use LWP::Simple;
+use strict;
+
+my $data = 
get('http://en.wikipedia.org/w/index.php?title=ISO_3166-1_alpha-2&action=raw');
+_utf8_on($data); # mess with Perl's internals... data is in UTF8, but we need 
to tell Perl that!
+my @lines = split /\r?\n/, $data;
+
+my $longest_name = 0;
+my $longest_link = 0;
+
+print "DROP TABLE Countries;\n";
+print "CREATE TABLE Countries (country varchar(2) PRIMARY KEY, country_name 
varchar(200), wikipedia_en varchar(120));\n";
+foreach (@lines)
+{
+       # toothpicks stood on end
+       if (/^  \s* \| \s*                      # Initial pipe symbol
+                       id=\"([A-Z]{2})\"       # ID attribute
+               \s* \| \s*                      # Another pipe symbol
+                       <tt>..<\/tt>            # Teletype font
+               \s* \|\| \s*                    # Double pipe symbol
+                       \[\[                    # Double opening brackets
+                               ([^\]\|]+)      # Link
+                               (\|([^\]]+))?   # Link text (optional)
+                       \]\]                    # Double closing bracket
+       /xi)
+       {
+
+
+               my $code = $1;
+               my $link = escape($2);
+               my $name = $4 || $2;
+
+               $link =~ s/\%20/_/g;
+               
+               $name =~ s/\'/\'\'/g;
+               
+               $longest_link = length($link) if length($link) > $longest_link;
+               $longest_name = length($name) if length($name) > $longest_name;
+               
+               print "INSERT INTO Countries VALUES ('$code', '$name', 
'http://en.wikipedia.org/wiki/$link');\n";
+       }
+}
+
+$longest_link += length('http://en.wikipedia.org/wiki/');
+
+print "-- Longest name: $longest_name\n";
+print "-- Longest link: $longest_link\n";
+print "\n";
+print "-- This needs to be piped into the postgresql command line client.\n";
+print "-- e.g.  ./countrycodes.pl | psql librefm\n";

Added: branches/stable/scripts/gobble.py
===================================================================
--- branches/stable/scripts/gobble.py                           (rev 0)
+++ branches/stable/scripts/gobble.py   2009-05-06 00:25:02 UTC (rev 1218)
@@ -0,0 +1,93 @@
+try:
+    import hashlib
+    md5hash = hashlib.md5
+except ImportError:
+    import md5
+    md5hash = md5.new
+import time
+from urllib import urlencode
+from urllib2 import urlopen
+
+
+class GobbleException(Exception):
+
+    pass
+
+
+class GobbleServer(object):
+
+    def __init__(self, server_name, username, password):
+        if server_name[:7] != "http://":
+            server_name = "http://%s"; % (server_name,)
+        self.name = server_name
+        self.password = password
+        self.post_data = []
+        self.session_id = None
+        self.submit_url = None
+        self.username = username
+        self._handshake()
+
+
+    def _handshake(self):
+        timestamp = int(time.time())
+        token = (md5hash(md5hash(self.password).hexdigest()
+                    + str(timestamp)).hexdigest())
+        auth_url = "%s/?hs=true&p=1.2&u=%s&t=%d&a=%s&c=imp" % (self.name,
+                                                               self.username,
+                                                               timestamp, 
token)
+        response = urlopen(auth_url).read()
+        lines = response.split("\n")
+        if lines[0] != "OK":
+            raise GobbleException("Server returned: %s" % (response,))
+        self.session_id = lines[1]
+        self.submit_url = lines[3]
+
+    def submit(self):
+        if len(self.post_data) == 0:
+            return
+        i = 0
+        data = []
+        for track in self.post_data:
+            data += track.get_tuples(i)
+            i += 1
+        data += [('s', self.session_id)]
+        response = urlopen(self.submit_url, urlencode(data)).read()
+        if response != "OK\n":
+            raise GobbleException("Server returned: %s" % (response,))
+        self.post_data = []
+        time.sleep(1)
+
+    def add_track(self, gobble_track):
+        i = len(self.post_data)
+        if i > 49:
+            self.submit()
+            i = 0
+        self.post_data.append(gobble_track)
+
+
+class GobbleTrack(object):
+
+    def __init__(self, artist, track, timestamp, album=None, length=None,
+                 tracknumber=None, mbid=None):
+        self.artist = artist
+        self.track = track
+        self.timestamp = timestamp
+        self.album = album
+        self.length = length
+        self.tracknumber = tracknumber
+        self.mbid = mbid
+
+    def get_tuples(self, i):
+        timestamp = str(int(time.mktime(self.timestamp.utctimetuple())))
+        data = []
+        data += [('a[%d]' % i, self.artist), ('t[%d]' % i, self.track),
+                 ('i[%d]' % i, timestamp)]
+        if self.album is not None:
+            data.append(('b[%d]' % i, self.album))
+        if self.length is not None:
+            data.append(('l[%d]' % i, self.length))
+        if self.tracknumber is not None:
+            data.append(('n[%d]' % i, self.tracknumber))
+        if self.mbid is not None:
+            data.append(('m[%d]' % i, self.mbid))
+        return data

Modified: branches/stable/scripts/import-jamendo.py
===================================================================
--- branches/stable/scripts/import-jamendo.py   2009-05-06 00:15:05 UTC (rev 
1217)
+++ branches/stable/scripts/import-jamendo.py   2009-05-06 00:25:02 UTC (rev 
1218)
@@ -211,7 +211,7 @@
 
                                                if 
self.track_exists(artist["name"], album["name"], track["name"]):
                                                        try:
-                                                               
self.cursor.execute("UPDATE Track SET downloadurl = %s, streamurl = %s, mbid = 
%s, license = %s, duration = %s, otherid = %s AND streamable = %s WHERE name = 
%s AND artist = %s AND album = %s", (track["downloadurl"], track["streamurl"], 
track["mbid"], track["license"], duration, otherid, streamable, track["name"], 
artist["name"], album["name"]))
+                                                               
self.cursor.execute("UPDATE Track SET downloadurl = %s, streamurl = %s, mbid = 
%s, license = %s, duration = %s, otherid = %s, streamable = %s WHERE name = %s 
AND artist = %s AND album = %s", (track["downloadurl"], track["streamurl"], 
track["mbid"], track["license"], duration, otherid, streamable, track["name"], 
artist["name"], album["name"]))
                                                                
self.conn.commit()
                                                        except Exception,  e:
                                                                
self.conn.rollback()

Added: branches/stable/scripts/most_recent.py
===================================================================
--- branches/stable/scripts/most_recent.py                              (rev 0)
+++ branches/stable/scripts/most_recent.py      2009-05-06 00:25:02 UTC (rev 
1218)
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+from datetime import datetime
+from optparse import OptionParser
+from urllib2 import urlopen
+
+from BeautifulSoup import BeautifulSoup
+
+
+if __name__ == '__main__':
+    usage = "%prog <USERNAME> <WEB SERVER>"
+    parser = OptionParser(usage=usage)
+    opts,args = parser.parse_args()
+    if len(args) != 2:
+        parser.error("All arguments are required.")
+
+    username,server = args
+    if server[:7] != "http://":
+        server = "http://%s"; % (server,)
+    soup = BeautifulSoup(urlopen('%s/user/%s/recent-tracks'
+                                    % (server, username)))
+    gobbles_list = soup.find('ul', {'class': 'gobbles'})
+    timestamp = gobbles_list.find('li')['about'].split('#')[1].split('.')[0]
+    print datetime.fromtimestamp(float(timestamp))

Added: branches/stable/scripts/offline-submit.py
===================================================================
--- branches/stable/scripts/offline-submit.py                           (rev 0)
+++ branches/stable/scripts/offline-submit.py   2009-05-06 00:25:02 UTC (rev 
1218)
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+import datetime
+import getpass
+from optparse import OptionParser
+import subprocess
+
+import mutagen
+from mutagen import easyid3
+
+from gobble import GobbleServer, GobbleTrack
+
+
+def _parse_date(string):
+    process = subprocess.Popen(['date', '-d %s' % (string,), '+%s'],
+                               stdout=subprocess.PIPE)
+    string = process.communicate()[0].strip()
+    return datetime.datetime.utcfromtimestamp(float(string))
+
+
+if __name__ == '__main__':
+    usage = "%prog <USERNAME> <SERVER> <START TIME> <MEDIA FILES>"
+    parser = OptionParser(usage=usage)
+    opts,args = parser.parse_args()
+    if len(args) < 4:
+        parser.error("All arguments are required.")
+
+    username,server,start_string = args[:3]
+    password = getpass.getpass()
+    tracks = args[3:]
+    server = GobbleServer(server, username, password)
+
+    dt = _parse_date(start_string)
+
+    for track in tracks:
+        f = mutagen.File(track)
+        if f is None:
+            raise Exception("%s caused problems." % (track,))
+        if isinstance(f, mutagen.mp3.MP3):
+            f = mutagen.mp3.MP3(track, ID3=easyid3.EasyID3)
+        title = f['title'][0]
+        artist = f['artist'][0]
+        length = f.info.length
+        album = f['album'][0]
+        tracknumber = f['tracknumber'][0]
+        t = GobbleTrack(artist, title, dt, album=album, length=length,
+                        tracknumber=tracknumber)
+        server.add_track(t)
+        dt += datetime.timedelta(seconds=length)
+    server.submit()

Modified: branches/stable/scripts/scrobbles-revisionism
===================================================================
--- branches/stable/scripts/scrobbles-revisionism       2009-05-06 00:15:05 UTC 
(rev 1217)
+++ branches/stable/scripts/scrobbles-revisionism       2009-05-06 00:25:02 UTC 
(rev 1218)
@@ -41,6 +41,9 @@
                        if "&gt;" in oldname:
                                newname = newname.replace("&gt;",">")
 
+                       if " (PREVIEW: buy it at www.magnatune.com)" in oldname:
+                               newname = newname.replace(" (PREVIEW: buy it at 
www.magnatune.com)","")
+
                        if oldname <> newname:
                                print "%s >>>INTO>>> %s" % (oldname, newname)
                                try:

Added: branches/stable/scripts/setstids
===================================================================
--- branches/stable/scripts/setstids                            (rev 0)
+++ branches/stable/scripts/setstids    2009-05-06 00:25:02 UTC (rev 1218)
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import sys;
+import adodb;
+
+class CleanUp:
+
+       def __init__(self, dbname, username):
+               self.conn = adodb.NewADOConnection('postgres')
+               self.conn.Connect('user=%s dbname=%s' % (username, dbname))
+
+       def doit(self):
+
+               self.cursor = self.conn.Execute('SELECT DISTINCT artist, album, 
track, mbid from Scrobbles WHERE stid is NULL');
+
+               while not self.cursor.EOF:
+                       row = self.cursor.GetRowAssoc(0)
+                       artist = row['artist']
+                       album = row['album']
+                       track = row['track']
+                       mbid = row['mbid']
+                       stid = self.lookupStid(artist,album,track,mbid)
+                       if not stid:
+                               tid = self.lookupTrack(artist, album, track, 
mbid)
+                               if tid:
+                                       print "Adding stid for %s/%s/%s/%s/%s" 
% (tid,artist,album,track,mbid)
+                                       self.acursor = 
self.conn.Execute("INSERT INTO Scrobble_Track (artist,album,name,mbid,track) 
VALUES(lower(%(artist)s),lower(%(album)s),lower(%(name)s),lower(%(mbid)s),%(track)s)",
 {'artist': artist, 'album': album, 'name': track, 'mbid': mbid, 'track': tid})
+                               else:
+                                       print "Adding tid for %s/%s/%s/%s" % 
(artist,album,track,mbid)
+                                       self.acursor = 
self.conn.Execute("INSERT INTO Track (artist,album,name,mbid) 
VALUES(%(artist)s,%(album)s,%(name)s,%(mbid)s)", {'artist': artist, 'album': 
album, 'name': track, 'mbid': mbid})
+                       else:
+                               print "Setting %d to match %s/%s/%s/%s" % 
(stid,artist,album,track,mbid)
+                               self.scursor = self.conn.Execute("UPDATE 
Scrobbles SET stid = %%(stid)s WHERE track = %%(track)s AND artist = 
%%(artist)s AND %s AND %s AND stid IS NULL" % (('lower(album) = 
lower(%(album)s)' if album else 'album IS NULL'), ('lower(mbid) = 
lower(%(mbid)s)' if mbid else 'mbid IS NULL')), {'stid': stid, 'track': track, 
'artist': artist, 'album': album, 'mbid': mbid})
+                       self.cursor.MoveNext()
+
+       def lookupStid(self, artist, album, track, mbid):
+               self.lucursor = self.conn.Execute('SELECT * from Scrobble_Track 
WHERE lower(artist) = lower(%%(artist)s) AND %s AND lower(name) = 
lower(%%(track)s) AND %s' % (('lower(album) = lower(%(album)s)' if album else 
'album IS NULL'), ('lower(mbid) = lower(%(mbid)s)' if mbid else 'mbid IS 
NULL')), {'artist': artist, 'album': album, 'track': track, 'mbid': mbid});
+               if not self.lucursor.EOF:
+                       lrow = self.lucursor.GetRowAssoc(0)
+                       return lrow['id']
+               else:
+                       return None
+
+       def lookupTrack(self, artist, album, track, mbid):
+               self.tucursor = self.conn.Execute('SELECT * from Track WHERE 
lower(artist) = lower(%%(artist)s) AND %s AND lower(name) = lower(%%(track)s) 
AND %s' % (('lower(album) = lower(%(album)s)' if album else 'album IS NULL'), 
('lower(mbid) = lower(%(mbid)s)' if mbid else 'mbid IS NULL')), {'artist': 
artist, 'album': album, 'track': track, 'mbid': mbid});
+               if not self.tucursor.EOF:
+                       trow = self.tucursor.GetRowAssoc(0)
+                       return trow['id']
+               else:
+                       return None
+
+       def close(self):
+               self.cursor.Close()
+               self.conn.Close() 
+
+if __name__ == "__main__":
+
+       if len(sys.argv) != 3:
+               print "Usage: setstids <pgsql dbname> <pgsql username>"
+               sys.exit(1)
+       
+       cleaner = CleanUp(sys.argv[1], sys.argv[2])
+       cleaner.doit()
+       cleaner.close()

Deleted: branches/stable/scripts/users.sql
===================================================================
--- branches/stable/scripts/users.sql   2009-05-06 00:15:05 UTC (rev 1217)
+++ branches/stable/scripts/users.sql   2009-05-06 00:25:02 UTC (rev 1218)
@@ -1 +0,0 @@
-SELECT username FROM `Users`

Modified: branches/stable/scripts/users_dump.sh
===================================================================
--- branches/stable/scripts/users_dump.sh       2009-05-06 00:15:05 UTC (rev 
1217)
+++ branches/stable/scripts/users_dump.sh       2009-05-06 00:25:02 UTC (rev 
1218)
@@ -2,6 +2,14 @@
 
 cd /home/librefm/turtle/data/wip || exit 1
 
+if [ -x /usr/bin/lockfile-create ]; then
+       lockfile-create /var/lock/gnukebox-userdump.lock
+       lockfile-touch /var/lock/gnukebox-userdump.lock &
+       LOCKER="$!"
+else
+       echo >&2 "Running without locking."
+fi
+
 #we rely on lack of whitespace here
 LIST=$(echo "SELECT DISTINCT username FROM Users;" | psql -q -t)
 
@@ -11,5 +19,10 @@
     
     echo 'COPY (SELECT * FROM Scrobbles where username='"'$I'"') TO STDOUT 
WITH CSV HEADER;' | psql -q >> $I.text.utf8
 
-    mv -- $I /home/librefm/turtle/data/
+    mv -- $I.text.utf8 /home/librefm/turtle/data/
 done
+
+if [ -x /usr/bin/lockfile-create ]; then
+       kill "${LOCKER}"
+       lockfile-remove /var/lock/gnukebox-userdump.lock
+fi





reply via email to

[Prev in Thread] Current Thread [Next in Thread]