So, this sort of works. The team at neo4j have made the iteration work for me... Usage:
>>python2.6 neo.py http://www.wherever.com
... and what it does it go to that web site and grabs a few links, following them, adding the pages and their data to a neo4j graph database.
import neo4j #See http://components.neo4j.org/neo4j.py/
import urllib2, traceback, re, urlparse, socket, sys, random, time
socket.setdefaulttimeout(4) #seconds
'''This is my attempt to begin to make proper classes and functions to talk to the neo4j database, it's not meant to be fancy or clever, I just want to be able to CRUD ok-ish'''
try:
db = neo4j.GraphDatabase("crawler_example3_db")
with db.transaction:
#How do we delete an index... which might have old nodes in? This doesn't work...any ideas... how do I empty an index, should I? Does deleting a node delete its refererence from the index
'''try:
for ref in db.index('pages'):
ref.delete()
except Exception, err:
print err
'''
pages = db.index("pages", create=True) # create an index called 'pages'
#print "Index created"
#return db, pages
except Exception, err:
print err
#print "db:", db
############# UTILITY FUNCTIONS #############
def get_links( data, url=''):
'I know I should use BeautifulSoup or lxml, but for simplicity it's a regex..ha'
links = [ ]
try:
found_links = re.findall(' href="?([^\s^"\'#]+)', data)
for link in found_links:
#print link
link = urlparse.urljoin(url, link)
link = link.replace("/..", "") #fix relative links
link = link.replace("'", "")
if link not in links and link[:7] == 'http://' :#avoid mailtos
links.append( link )
except Exception, err:
print err
return links
def wget(url):
try:
handle = urllib2.urlopen(url)
if 'text/html' in handle.headers['content-type']:
data = unicode(handle.read(), errors='ignore')
return data
except Exception, e:
print "Error wgetting", e, url
#print traceback.print_exc()
return None
def delete_all_pages():
with db.transaction:
try:
for node in db.node:
id = node.id
node.delete( )
print id, "deleted"
except Exception, err:
print err
pass # fails on last iteration?
print "All pages deleted!"
def create_page( url, code=200 , follow=True):
'''this creates a page node then adds it to the "pages" index, if it's there already it'll get it'''
# Get the actual page data from the web...
data = wget( url ) #get the actual html
if not data:
return None
print "Boo!", url
else:
data_len = len(data)
page_node = pages[url] # does this page exist yet?
if not page_node:
page_node = db.node(url=url, code=code, data=str(data), data_len=data_len) # create a page
pages[ url ] = page_node # Add to index
print "Created:" , url
else:
print "Exists already:" , url
#Now create a page for every link in the page
if follow == True:
print "\tfollowing links"
i = 0
links = get_links(data, url)
print len( links ) , "found"
if len(links) > 20:
links = links[:19]
for link in links:
print "\tcreating:", link
create_page( link, follow=False)
return page_node
def get_one(url):
'given a url with return a node obj'
with db.transaction:
try:
node = pages [ url ]
if node == None:
print "Node is none! Creating..."
create_page( url, code=200 , follow=False)
return node
except Exception, err:
print err
return None
def list_all_pages( ):
# Just iterate through the pages to make sure the data in in there...
print "Listing all pages..."
with db.transaction:
for node in db.node:
try:
print node
#print node['data'][:150], "..."
print
except Exception, err:
print err
print "...done listing!"
def delete_one( url ):
''
with db.transaction:
try:
node = pages[ url ]
if node:
node.delete( )
print "Node with id=", node.id, "deleted"
#delete from index
del pages[url]
except Exception, err:
print "Node probably not found:", err
print dir(node) # let's have a look
def find_links_between_pages( ):
#'Just iterate through the pages to make sure the data in in there...'
print "Linking all pages..."
with db.transaction:
try:
for node in db.node:
try:
print str(node), node['url'], node['data_len']
links = get_links( node['data'], node['url'])
for link in links:
try:
print link
#look to see if a node with that url exists, if it doesn't it's created...
other_node = get_one( link )
if not other_node:
pass
else:
other_node.links_to( node )
except Exception, err:
print err
print
except Exception, err:
print err
print
except Exception, err:
pass # fails on last iteration?
print err
class Backlink(neo4j.Traversal):
types = [neo4j.Outgoing.links_to]
order = neo4j.DEPTH_FIRST
stop = neo4j.STOP_AT_END_OF_GRAPH
def isReturnable(self, pos):
if pos.is_start: return False
else:return pos.last_relationship.type == 'links_to'
if __name__ == '__main__':
#### LET'S GET STARTED ######
try:
sys.argv[1]
url = sys.argv[1]
except:
print "No url passed, using 'http://diveintopython.org/'"
url = 'http://diveintopython.org/'
print "Starting:", url
######## DELETE ALL THE PAGES WE HAVE SO FAR ############
# Avoiding this because of ...
#jpype._jexception.RuntimeExceptionPyRaisable: org.neo4j.graphdb.NotFoundException: Node[0]
#... errors later...
#print "Deleting existing pages..."
#delete_all_pages( ) #we may want to add new data to each page... forget this for now
######## ADD SOME ACTUAL PAGES #################
#print "Creating some pages "
with db.transaction:
create_page( url, 200, follow=True ) #Also fetches some pages linked from this page
######## NOW GET SOME NODES OUT ###############
print
print "Has our data made it to the database? Let's see..."
# Do some fishing for nodes...
with db.transaction:
try:
node = get_one( url )
print '\tget one:' + str( node ) , "... oh yes!"
print "\tid:", node.id , "url:", node['url'], "data-length:", node['data_len']
print
#This should fail
#print "This SHOULD fail.."
#node = get_one( 'http://www.pythonware.com/daily/' )
#print 'get one:' + str( node )
#print "id:", node.id , "url:", node['url'], "data length:", node['data_len']
#print
except Exception, err:
print "Probably not a node with that URL"
print err
print
######### TRY TO ITERATE ALL PAGES FETCHED ################
list_all_pages( )
######### TRY TO DELETE ONE ################
#delete_one( url )
# Now let's see if it has gone
#list_all_pages( ) #or maybe later
######### LET'S LOOK FOR RELATIONSHIPS BETWEEN PAGES ################
print
print "Doing linking pages..."
find_links_between_pages( ) #goes to every page, looking for other pages in the database
print
print "shutting down, saving all the changes, etc"
db.shutdown()
