Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Some comments and trying different for loop for adding requests

Browse Source
This commit is contained in:
Nout van Deijck 2014-04-23 13:48:44 +02:00
parent 1ced65e2b6
commit 3e1b33164e
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
FourmiCrawler/parsers/WikipediaParser.py
View File

@ -50,10 +50,18 @@ class WikipediaParser(Parser):
# request=Request(self.getchemspider(sel)) # request=Request(self.getchemspider(sel))
# itemlist.append(request) # itemlist.append(request)
for identifier in self.get_identifiers(sel):
request_identifier=Request(identifier) identifiers=self.get_identifiers(sel)
# print request_identifier # print identifiers
itemlist.append(request_identifier)
for i, identifier in enumerate(identifiers):
request = Request(identifier)
print request
# for identifier in self.get_identifiers(sel):
# request_identifier=Request(identifier)
# # print request_identifier
# itemlist.append(request_identifier)
return itemlist return itemlist
@ -79,9 +87,6 @@ class WikipediaParser(Parser):
def get_identifiers(self, sel): def get_identifiers(self, sel):
links=sel.xpath('//span[contains(concat(" ",normalize-space(@class)," "),"reflink")]/a[contains(concat(" ",normalize-space(@class)," "),"external")]/@href').extract() links=sel.xpath('//span[contains(concat(" ",normalize-space(@class)," "),"reflink")]/a[contains(concat(" ",normalize-space(@class)," "),"external")]/@href').extract()
# identifiers=[]
# for link in links: print links
# identifier=Request(link)
# identifiers.append(identifier)
# print identifiers
return links return links