# author: Jesse Hoey # School of Computer Science # University of Waterloo # Date: June 2, 2012 # Please feel free to use this code for any academic or not-for profit usages. # This code carries no guarantees, either explicity or implicit. # Please make sure you check these results before using them, as I am not # responsible for any errors that are made. # Use this script wisely - see NOTE below on running times # # this script will query Google scholar using a paper title, will recover # the bibliographic information, and then check all citations against the # authors in the author list, removing any that are by any author. # the final result is shown for each paper, along with some other stuff. # # if you don't see bibtex entries, then you may need to set a cookie somewhere # to get google scholar to include this data, as it doesn't seem to be something # you can add to the URL # # This script does not (yet) check to see if the top paper returned is the same # as the one you are looking for, it just assumes so and returns all citations # for that paper. It does, however, also show the bibtex for it so you can # check manually. # TO DO: modify the script so it takes in a list of authors+titles, and # check that the returned result matches up # # NOTE: this sript is VERY VERY SLOW, but this is on purpose, as anything # faster will set off alarm bells at Google and they will deny you further access. # set it running somewhere, go and get a beverage, and come back in a few hours # (or a few days, depending on how many publications you have) # if you want to speed it up, try changing the sleep constants below import urllib, urllib2, cookielib, urllister, sys import time import random #sleep amounts for when getting papers basesleep=60 randsleep=60 # sleep amounts for when getting bibtex entries # speed this up as much as possible otherwise this really takes forever basesleep_bib=10 randsleep_bib=15 def get_page(url): filename = "cookies.txt" headers = { #'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12' 'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0' } request = urllib2.Request(url, None, headers) cookies = cookielib.MozillaCookieJar(filename, None, None) cookies.load() cookie_handler= urllib2.HTTPCookieProcessor(cookies) redirect_handler= urllib2.HTTPRedirectHandler() opener = urllib2.build_opener(redirect_handler,cookie_handler) response = opener.open(request) return response.read() def get_bibtex(citationlink): cparts=citationlink.split("?") startcite=0 totalnumcites=0 biburls=[] # keep looping over the returned pages until # we're out of pages of citations while True: params = "start="+str(startcite)+"&"+cparts[1] url = base_url+cparts[0]+"?"+params page_data = get_page(url) parser = urllister.URLLister() parser.feed(page_data) parser.close() numcites=0 for url in parser.urls: if url.find('scholar.bib')>0: numcites+=1 biburls.append(url) sys.stdout.flush() time.sleep(random.random()*randsleep_bib+basesleep_bib) if numcites==0: break else: startcite=startcite+numcites totalnumcites+=numcites numcites=0 print 'total num citations',totalnumcites return biburls def extractAuthors(bibpage): rr=bibpage[bibpage.find("author"):] r2=rr[:rr.find("}")] r3=r2[(r2.find("{")+1):] r4=r3.split("and") r5=[] for r in r4: r2=r.strip() r3=r2.split(",") r5.append(r3[0]) return r5 # function to get the citations list for a page termstring # we do this by constructing the search query with ?q=termsting # and then parsing the returned file looking at the first one # def get_cites(termstring, base_url, scholar_subdir): terms=termstring.rsplit(" ") # print terms searchstring="\""+" ".join(terms).strip()+"\"" # print searchstring params = urllib.urlencode({'q': searchstring}) url = base_url+url_subdir+"?"+params # print url page_data = get_page(url) #print page_data parser = urllister.URLLister() parser.feed(page_data) parser.close() # we only want the top result's citations, which may be missing (meaning there are none) citeurl=None citedauthors=None # parser.urls contains a list of all urls on the page. now we look only at the top one # get the citation number (denoted with ?cites=) # and wait to see the bibtex entry (we assume this will be there) # once we see that (print it) and we are done for url in parser.urls: if url.find('scholar.bib')>0: #this is the last link, so if there was a citationurl (denoted with scholar.bib), #we return it otherwise return none retpage=get_page(base_url+url) print "citation google has is as follows: " print retpage #now we really want to extract the author names from this, and use it below when we check for self-citations ***** citedauthors=extractAuthors(retpage) #also want to compare the returned title and authors to the paper title we are searching for in termstring #termstring should really be called titlestring, and we could also pass in the authors read from the database or titles.txt file #if the data does not compare, it means Google has found the wrong paper. We can then either look down the list #of URLS for another one, to see if any on the first page compare #if none do ... we ?? # or could just output the title Google has along with an error flag and the normal citation count - # this can then be manually checked against the title on file sys.stdout.flush() break elif url.find('scholar?cites')>0: foundcitationurl=True citeurl=url return citeurl,citedauthors def anySelfCite(retpage,citedauthors): for citedauthor in citedauthors: if retpage.find(citedauthor)>0: return True return False if __name__ == "__main__": if len(sys.argv)==2: base_url = "http://scholar.google.ca" url_subdir = "/scholar" titles_file=sys.argv[1] file = open(titles_file) #could replace this with a database read - read the titles of all papers, as well as authors # then below, loop through all titles in the file, and check that the title returned by google is the correct one (agrees on most characters?) # could also check the author list to make sure it compares properly lines=[] while 1: termstring=file.readline() print 'title of paper is ',termstring sys.stdout.flush() if not termstring: break else: # get the file citations page first ***** papercitationlink,citedauthors=get_cites(termstring,base_url, url_subdir) numcitations=0 numselfcitations=0 if not papercitationlink == None: # get the bibtex for all papers that cite this one biburls=get_bibtex(papercitationlink) #now loop over all papers that cite the one we're interested in #and remove all those with an author (self-citations) for bib in biburls: retpage=get_page(base_url+bib) #print retpage numcitations+=1 # should either get the authors from the titles.txt file or from the bibtex file in get_cites above #****** if anySelfCite(retpage,citedauthors): #print "self citation!!!" numselfcitations+=1 time.sleep(random.random()*randsleep+basesleep) #print "numcitations: ",numcitations," num self-cites: ",numselfcitations sys.stdout.flush() print "number of citations for paper: ",termstring," is: ",numcitations," of which ",numselfcitations," are self, leaving ",(numcitations-numselfcitations)," actual citations" sys.stdout.flush() else: print "usage: python "+sys.argv[0]+" titles.txt\n titles.txt must contain a list of publication titles."