scopus/scopus-get-publications.py

# Get new publications
# Publication and citations retrieval
# A. Kopmann, 6.2.17 (ak)
#
# Scope:
# Publications are once added to wordpressas a post or comment.
# Afterwards scopus will not change or modify anything any more !!!
# Update is completely in the resonsibility of the ufo users
#

# Todo:
# - add mail to author button
# - save full scopus data in the database
#

# Configuration - Scopus

import datetime
import requests
import json
import os.path

from my_scopus import MY_API_KEY
from my_scopus import ak, csa, pdv, ufo, ufo_pdv, ufo_ips, ufo_eps, ufo_apps
from ak_scopus import get_scopus_list, get_scopus_data, get_scopus_refs


from ak_wordpress import wordpress_post_by_scopus, wordpress_comment_by_scopus, wordpress_get_post


# Mysql persistent data (Accout: scopus, $scopus$)
import pymysql.cursors
import pymysql

db_host = 'localhost'
db_user = 'scopus'
db_pw = '$scopus$'
db_name = 'scopus'

log_file = '/Users/kopmann/scopus-publications.log'


# Summary
npubs = 0
nnewpubs= 0
ncites = 0
nnewcites = 0



# Read publications of a list of authors and store in the database
def update_publications(authids,authname='',scopus_opts = '',max=0):


    print "=== Update of publications for the author group: " + authname
    #print str(authids)


    # Connect to the database
    connection = pymysql.connect(host=db_host,user=db_user,password=db_pw,db=db_name,charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor)

    # Request all publications of a list of authors (in one query)
    # Result: list of records with (scopus ids, eid, citedbycount)
    # The citation could be used later also by wordpress (may be via a plugin)

    publist = get_scopus_list(authids,scopus_opts,max)
    #publist = get_scopus_list(authids, scopus_opts, 3)
    #publist = get_scopus_list(authids, '(PUBYEAR AFT 2014)')
    print "Total number of publications: %d" % len(publist)
    #print publist


    # Save all publication to the publication database
    try:
        with connection.cursor() as cursor:
            for pub in publist:
                # 1 / Create a new records
                #print pub # Todo: strip the prefix SCOPUS_ID?!
                sql = "INSERT IGNORE INTO `publications` (`scopusid`,`eid`) VALUES (%s,%s)"
                cursor.execute(sql, (pub[0],pub[1]))
                    
                sql = "UPDATE `publications` SET `citedbycount` = %s WHERE `scopusid` = %s"
                cursor.execute(sql, (pub[2],pub[0]))

                # 2 / Add categories
                if len(authname) > 0:
                    catlist = []
                    sql = "SELECT categories FROM publications WHERE scopusid = %s"
                    cursor.execute(sql, (pub[0]))
                    result = cursor.fetchall()
                    if len(result) > 0:
                        #print "Categories %s" % result[0]['categories']
                        cat = result[0]['categories']

                    try:
                        catlist = json.loads(cat)
                    except TypeError:
                        print("No categories upto now")

                    if authname not in catlist:
                        catlist += [authname]
                
                    sql = "UPDATE `publications` SET `categories` = %s WHERE `scopusid` = %s"
                    cursor.execute(sql, (json.dumps(catlist),pub[0]))
                        

            # connection is not autocommit by default. So you must commit to save
            # your changes.
            connection.commit()

    finally:
        connection.close()


# Read all citations and store in the citation table
def update_citations():
    global npubs
    
    print ""
    print "=== Update citatation of all publication in the database"
    
    # Connect to the database
    connection = pymysql.connect(host=db_host,user=db_user,password=db_pw,db=db_name,charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor)

    # Loop over the publications and read all citations from scopus

    # Todo: Shift to a separate script !?
    try:
        with connection.cursor() as cursor:
            # Read a single record
            sql = "SELECT wpid,eid,citedbycount,citesloaded FROM publications WHERE wpid > 0"
            cursor.execute(sql)
            result = cursor.fetchall()
          
            print "Total number of publications is %d" % len(result)
            npubs = len(result)
            #print "Npubs = %d" % npubs

            for pub in result:
                wpid = int(pub['wpid'])
                if pub['citedbycount'] is None:
                    citedbycount = 0
                else:
                    citedbycount = int(pub['citedbycount'])
                if pub['citesloaded'] is None:
                    citesloaded = 0
                else:
                    citesloaded = int(pub['citesloaded'])

                # read list of citations
                if pub['eid'] and (citedbycount > citesloaded):
                    
                    print "Processing %d = %s previously cited by %d"  % (wpid, pub['eid'], citesloaded)

                    data = get_scopus_refs(pub['eid'])
                    #print json.dumps(data,sort_keys=True,indent=4, separators=(',', ': '))
                    
                    n = len(data)
                    #print "Number of citations loaded for processing %d" % n
                    #print data
                    
                    if n > 0:
                        for pub in data:
                            #print pub['eid'] + '  ' + pub['dc:title']
                           
                            try:
                                pubstr = json.dumps(pub)
                            except TypeError:
                                print("Error serializing pub entry")

                            # save all comments to the database
                            # wirte complete scopus data of the article !?
                            sql = "INSERT IGNORE INTO `citations` (`wpid`,`scopusid`,`eid`,`scopusdata`) VALUES (%s,%s,%s,%s)"
                            cursor.execute(sql, (wpid,pub['dc:identifier'],pub['eid'],pubstr))
                            connection.commit()

                        # Update the number of cites for this article
                        if n > citesloaded:
                            print "New citations found %d -> %d" %(citesloaded,n)
                            sql = "UPDATE `publications` SET `citesloaded`=" + str(n) + " WHERE wpid=" + str(wpid)
                            #print sql
                            cursor.execute(sql)
                            connection.commit()


    finally:
        connection.close()




# Create wordpress posts for all entries that have none
def update_wp_posts():
    global nnewpubs

    print ""
    print "=== Create posts for newly registered publication in scopus"

    # Connect to the database
    connection = pymysql.connect(host=db_host,user=db_user,password=db_pw,db=db_name,charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor)
        

    # Todo: Shift to a separate script !?
    try:
        with connection.cursor() as cursor:
            # Count all publications
            #sql = "SELECT COUNT(id) FROM publications"
            #cursor.execute(sql)
            #result = cursor.fetchall()
            #if len(result) > 0:
            #print result[0]['COUNT(id)']

            # Read a single record
            sql = "SELECT scopusid,categories FROM publications WHERE wpid IS NULL"
            cursor.execute(sql)
            result = cursor.fetchall()
            if len(result) > 0:
                print "Number of new publications is %d" % len(result)
                nnewpubs = len(result)
            else:
                print "Nothing new found"


        # Retrieve all information required for the wordpress page
        for pub in result:
            print "Processing " + pub['scopusid'] + " categories " + pub['categories']

            data = get_scopus_data(pub['scopusid'])
            #print json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))

            # Parse categories
            catlist = []
            try:
                catlist = json.loads(pub['categories'])
            except TypeError:
                print("No categories specified")

            wpid = wordpress_post_by_scopus(data, catlist)
            
            #print wpid
            #print pub['scopusid']

            # Warning: the resulting string uses double quotes (") so use
            # single quotes (') for the sql command
            datastr = json.dumps(data)
            #print datastr

            # Update publication database !!!
            with connection.cursor() as cursor:
                # Read a single record
                #sql = "UPDATE publications SET wpid=" + str(wpid) + ",scopusdata='" + datastr + "' WHERE scopusid = '" + pub['scopusid'] + "'"
                sql = "UPDATE publications SET wpid=" + str(wpid) + " WHERE scopusid = '" + pub['scopusid'] + "'"
                cursor.execute(sql)
                connection.commit()

    finally:
        connection.close()


def update_wp_comments():
    global ncites
    global nnewcites
    
    print ""
    print "=== Create comments for newly registered citations in scopus"
    
    
    # Connect to the database
    connection = pymysql.connect(host=db_host,user=db_user,password=db_pw,db=db_name,charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor)
    
    
    # Todo: Shift to a separate script !?
    try:
        with connection.cursor() as cursor:
            # Count all citations
            sql = "SELECT COUNT(id) FROM citations"
            cursor.execute(sql)
            result = cursor.fetchall()
            if len(result) > 0:
                ncites = result[0]['COUNT(id)']
                #print result[0]['COUNT(id)']

            # Read a single record
            sql = "SELECT id,wpid,scopusdata FROM citations WHERE wpcommentid IS NULL"
            cursor.execute(sql)
            result = cursor.fetchall()
            print "Number of new citations is %d" % len(result)
            nnewcites = len(result)

            for pub in result:
                wpid = int(pub['wpid'])
                print "Processing post " + str(wpid)
            
                data = []
                try:
                    data = json.loads(pub['scopusdata'])
                except TypeError:
                    print("Scopus data missing?!")

                # If the creation of the comment fails, the wpcommentid 0 is
                # written to the database. This means, there is no second try
                # to get this citations added.
                # All failed comments can be found by searching for wpcommentid = 0
                #
                wpcommentid = 0
                try:
                    wpcommentid = wordpress_comment_by_scopus(wpid, data)
                except:
                    print "Error: Submission of comment failed"

                sql = "UPDATE citations SET wpcommentid=" + str(wpcommentid) + " WHERE id = '" + str(pub['id']) + "'"
                cursor.execute(sql)
                connection.commit()

    finally:
        connection.close()


# Todo: Add a script to save the data for all publications in the database!!!
# There was some problem before?!
#

# Todo: Add scripts to check consistence in the database
# and fix problems if detected
# E.g. search for wpcommentid == 0
# Check if, wp posts + comments are still availabe, display
# deleted entries
#



# Main

start = datetime.datetime.now()

print ""
print "***********************************************"
print "**** scopus-get-publications / " + start.strftime("%Y-%m-%d") + " *****"
print "***********************************************"
print ""



# Update publaction database; search for new publications
# Loop over all user groups defined in ak_scopus.py

# Todo: Detect, if there is no access to scopus !!!
#


# Define the author, that should be considered
#authors = ["Computing", ufo_pdv]
#print authors

#update_publications(ufo_pdv, "Computing", '(PUBYEAR AFT 2006)',10)
update_publications(ufo_pdv, "Computing", '(PUBYEAR AFT 2006)')
update_publications(ufo_ips, "X-ray Imaging", '(PUBYEAR AFT 2010)')
update_publications(ufo_eps, "Electronics", '(PUBYEAR AFT 2010)')
update_publications(ufo_apps, "Morphology", '(PUBYEAR AFT 2010)')


update_wp_posts()

# read all citations
# Todo: read only new citations?!


update_citations()


# loop over all cites and post comments to wordpress, when necessary
# update database

update_wp_comments()
# Todo: deactivate comments for scopus posts!!!


# Display summary
end = datetime.datetime.now()
print ""
print "Summary: (see also logfile %s) " % log_file
print "Date       = " + str(start)
print "NPubs      = " + str(npubs)
print "NNewPubs   = " + str(nnewpubs)
print "NCites     = " + str(ncites)
print "NNewCites  = " + str(nnewcites)
print "Runtime    = " + str(end - start)


# Write summary to log file
if not os.path.isfile(log_file):
    print "Create logfile " + log_file
    # Open file and write header
    log = open(log_file,"w")
    log.write(__file__ + "\n")
    log.write("\n")
    log.write("      Date\t    Time\tNPubs\tNNewP\tNCite\tNNewC\t          TRun\n")
    log.write("------------------------------------------------------------------------------\n")
    log.close()


log = open(log_file,"a")
log.write("%s\t%s\t%5d\t%5d\t%5d\t%5d\t%s\n" % (start.strftime("%Y-%m-%d"),
                                    start.strftime("%H:%M:%S"),
                                    npubs,nnewpubs,ncites,nnewcites,str(end-start)))
log.close()



# done