so i have changed what i what form my recipe. i will try to write the full version in pure python later, but now i want to do this as a recipe.
if you take a look
here you will see a list links on the page. i want the article title to be "the 1st link text" - "the 2nd link text"
right now it is just "the 2nd link next". the id of the 1st link is "CompNmHref" i just dont know how to do it with the for loop and soup. is there a "tag before" command in soup? because we are talikng about the tag befor the "item" in my code...
Spoiler:
Code:
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, re
class AlisonB(BasicNewsRecipe):
title ='Maya'
__author__ = 'marbs'
description = 'blah'
language = 'en'
no_stylesheets = True
publisher = 'marbs'
# simultaneous_downloads = 1
# delay = 25
category = 'column'
extra_css='body{direction: rtl;} .article_description{direction: rtl; } a.article{direction: rtl; } .calibre_feed_description{direction: rtl; }'
# no_stylesheets = True
use_embedded_content= False
remove_attributes = ['width','height']
no_stylesheets = True
oldest_article = 24
remove_javascript = True
remove_empty_feeds = True
max_articles_per_feed = 5000
INDEX = 'http://maya.tase.co.il/'
def parse_index(self):
feeds = []
for title, url in [
#(u"Feed", u"http://maya.tase.co.il/bursa/index.asp?view=search&company_group=3000&arg_comp=&srh_comp_lb=1007&srh_from=2010-01-01&srh_until=2010-09-28&srh_anaf=-1&srh_event=9999&is_urgent=0&srh_company_press="),
(u"הודעות מאתמול", u"http://maya.tase.co.il/bursa/index.asp?view=yesterday"),
]:
articles = self.make_links(url)
if articles:
feeds.append((title, articles))
return feeds
def make_links(self, url):
title = 'Temp'
current_articles = []
soup = self.index_to_soup(url)
print 'url is', url
# print 'The soup is: ', soup
for item in soup.findAll('a',attrs={'class':'A3'}):
print 'item is: ',item
#link = item.find('a')
#titlecheck = self.tag_to_string(link)
#url_test = re.search('javascript', item['href'])
if not re.search('javascript', item['href']):
temp2= self.INDEX + 'bursa/' + item['href']
# temp2=[temp3]
print 'url1 is', temp2
soup1 = self.index_to_soup(temp2)
# print 'the new soup is', soup1
print '6714'
for item1 in soup1.findAll('iframe'):
print 'item1 is:' , item1
txt= item1['src']
print 'FOUND GOOD URL'
re1='.*?' # Non-greedy match on filler
re2='(mayafiles)' # Variable Name 1
re3='(.)' # Any Single Character 1
re4='.*?' # Non-greedy match on filler
re5='htm' # Uninteresting: word
re6='.*?' # Non-greedy match on filler
re7='(htm)' # Word 1
rg = re.compile(re1+re2+re3+re4+re5+re6+re7,re.IGNORECASE|re.DOTALL)
m = rg.search(txt)
if m:
var1=m.group(1)
c1=m.group(2)
word1=m.group(3)
print "("+var1+")"+"("+c1+")"+"("+word1+")"+"\n"
url = item1['src']
else:
url = 'http://www.pdfdownload.org/pdf2html/pdf2html.php?url=' + item1['src'] + '&images=yes'
print 'url is: ', url
title = self.tag_to_string(item)
print 'title is: ', title
current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) # append all this
return current_articles