とりあえず過去ログ分ではうまくいき一段落したので公開してみる。
感想としては、不定型のHTMLのパースはもう嫌。見た目定型に見えても、idを付けてあるタグが違ったり、idの内容のフォーマットが途中から変わったり、要素が必ずしもタグで囲まれているわけではなかったり。こういうパターンなのかと書いてはエラーがでて修正しての繰り返し。面倒すぎる。
HTMLのパースにBeautifulSoup使ってるのに、RSSの出力はwriteで完全手作業なのはいまいち名前空間の扱いとかが分からなかったから。とりあえずバリデータは通ったからよしとする。誰かいいやつ知っていたら教えてください。
RSS Validator (feedAnalyzer)
で、なんでこんなことをしたかというと手持ちのウォークマンで聞くため。
#!/usr/bin/env python # -*- coding: utf-8 -*- import codecs from BeautifulSoup import BeautifulSoup as BS from sqlobject import * import os import datetime import urllib2 import email import email.utils import time import re import xml.sax.saxutils as saxutils galge_url = "http://www.galge.com/radio/galge/" rss_path = "/var/www/html/podcast/galge.rdf" rss_source = "/var/www/html/podcast/galge.sqlite" class LocalTimeZone(datetime.tzinfo): def __isdst(self, dt): tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) stamp = time.mktime(tt) tt = time.localtime(stamp) return tt.tm_isdst > 0 def utcoffset(self, dt=None): if dt: if self.__isdst(dt): return datetime.timedelta(seconds=-time.altzone) return datetime.timedelta(seconds=-time.timezone) def dst(self, dt=None): return datetime.timedelta(seconds=-time.altzone+time.timezone) def tzname(self, dt=None): if dt: return time.tzname[self.__isdst(dt)] return time.tzname[0] def date_rfc2822(dt): if not dt.tzinfo: tzdelta = LocalTimeZone().utcoffset ().seconds else: tzdelta = dt.tzinfo.utcoffset ().seconds dt_tz = datetime.datetime.timetuple (dt) + (tzdelta,) return email.utils.formatdate(email.utils.mktime_tz(dt_tz)) class EntryList(SQLObject): title = StringCol(notNone=True) subtitle = StringCol() url = StringCol(notNone=True, unique=True) subtitle = StringCol() url = StringCol(notNone=True, unique=True) datatype = StringCol() summary = StringCol() date = DateTimeCol() image = StringCol() sqlhub.processConnection = connectionForURI("sqlite:"+rss_source) EntryList.createTable(ifNotExists=True) def abspath(path): return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) def toRSS(outfilepath): out = codecs.open(abspath(outfilepath),"w","utf-8") summary = u"""”金田まひる・倉田まりや”のGalge.comラジオは、美少女ゲームやフィギュア、アニメなど様々な情報をお伝えするポータルサイト・Galge.comがプロデュースするフリーダムなWebラジオです。皆さんから寄せられた情報を御紹介したり、様々な場所に突撃したりと、ユーザーの皆様と一緒に作る楽しい番組を目指しています！！""" title = u"金田まひる・倉田まりやのGalge.comラジオ" link = "http://www.galge.com/radio/galge/#radio" copyright = u"(c) 2008 Galge.com / 井桁屋" image = "http://www.galge.com/radio/galge/images/Galge_radio03.jpg" out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n") out.write("""<rss xmlns:itunes=\"http://www.itunes.com/dtds/podcast-1.0.dtd\" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" version=\"2.0\">\n""") out.write("<channel>\n") out.write("\t<title>" + saxutils.escape(title) + "</title>\n") out.write("\t<link>" + link + "</link>\n") out.write("\t<language>" + "ja" + "</language>\n") out.write("\t<copyright>" + saxutils.escape(copyright) + "</copyright>\n") out.write("\t<itunes:subtitle>" + saxutils.escape(title) + "</itunes:subtitle>\n") out.write("\t<description>" + saxutils.escape(summary) + "</description>\n") out.write("\t<itunes:summary>" + saxutils.escape(summary) + "</itunes:summary>\n") out.write("\t<itunes:image href=\"" + image + "\"></itunes:image>\n") out.write("\t<lastBuildDate>" + date_rfc2822( datetime.datetime.fromtimestamp(time.time())) + "</lastBuildDate>\n") for ent in EntryList.selectBy().orderBy("date"): out.write("\t<item>") out.write("\t\t<title>" + saxutils.escape(ent.title.decode("utf-8")) + "</title>\n") out.write("\t\t<link>" + ent.url + "</link>\n") if ent.subtitle: out.write("\t\t<itunes:subtitle>" + saxutils.escape(ent.subtitle.decode("utf-8")) + "</itunes:subtitle>\n") if ent.summary: + "</itunes:subtitle>\n") if ent.summary: out.write("\t\t<itunes:summary>" + saxutils.escape(ent.summary.decode("utf-8")) + "</itunes:summary>\n") out.write("\t\t<enclosure url=\"" + ent.url + "\" type=\"" + ent.datatype + "\" length=\"1000\"></enclosure>\n") if ent.image: out.write("\t\t<itunes:image href=\"" + ent.image + "\"></itunes:image>\n") out.write("\t\t<guid>" + ent.url + "</guid>\n") out.write("\t\t<pubDate>" + date_rfc2822(ent.date) + "</pubDate>\n") out.write("\t</item>\n") out.write("</channel>\n") out.write("</rss>\n") def importBackNumber(start=1, end=110): starturl = (start // 20) + 1 endurl = (end // 20) + 1 mp3pat = re.compile(".*\.mp3$") titlepat = re.compile(u"[0-9]{2,3}",re.M) baseurl = "http://www.galge.com/radio/galge/" now_no = start conn = sqlhub.getConnection() trans = conn.transaction() trans.rollback() trans.begin() try: for c in range(starturl, endurl + 1): html = urllib2.urlopen( "http://www.galge.com/radio/galge/back%02d.html"%c) bs = BS(html.read()) for node in bs.findAll(): node.attrMap = dict(node.attrs) end_no = c * 20 + 1 if end_no > (end + 1): end_no = end + 1 for i in range(now_no, end_no): if i < 81: content = bs.findAll("a", {"name" : "%02d"%i})[0] elif i == 86: content = bs.findAll(attrs={"id" : "%02d"%96})[1] elif i == 101: content = bs.findAll(attrs={"id" : "n102"})[1] elif i < 101: content = bs.findAll(attrs={"id" : "%02d"%i})[0] else: content = bs.findAll(attrs={"id" : "n%03d"%i})[0] while True: if content.findAll("img", {"width" : 200, "border" : 1}): flag = False for img in content.findAll("img", {"width":200, "border":1}): for img in content.findAll("img", {"width":200, "border":1}): if img.attrMap["src"].rfind("galge0") > 0: flag = True break if flag: break content = content.parent title = content.h4.contents[0] if i < 81: subtitle = content.findAll("a", {"name" : "%02d"%i})[0].next elif i < 104: subtitle = content.find("table", {"class" : "txt12k", "width" : 340}).previous.previous.previous else: subtitle = content.next if isinstance(subtitle, basestring): subtitle = subtitle.replace("\r\n","").replace(" ","") else: subtitle = None if i < 44: summary = content.h4.next.next\ .replace("\r\n","").replace(" ","") summary = summary + "\n" summary = summary + content.h4.next.next.next.next\ .replace("\r\n","").replace(" ","") elif i < 104: summary = content.h4.next.next.next.next.next\ .replace("\r\n","").replace(" ","") summary = summary + "\n" summary = summary + \ content.h4.next.next.next.next.next.next.next\ .replace("\r\n","").replace(" ","") else: summary = content.h4.next.next\ .replace("\r\n","").replace(" ","") summary = summary + "\n" summary = summary + content.h4.next.next.next.next\ .replace("\r\n","").replace(" ","") for atag in content.findAll("a"): try: url = atag.attrMap["href"] except Exception: continue if mp3pat.match(url): break date_src = urllib2.urlopen(url).headers\ .getheader('Last-Modified') if date_src: date = datetime.datetime.fromtimestamp(time.mktime( email.utils.parsedate(date_src))) else: email.utils.parsedate(date_src))) else: date = None for img in content.findAll("img", {"width" : 200, "border" : 1}): image = img.attrMap["src"] if image.rfind("galge0") > 0: image = baseurl + image if isinstance(title, unicode): title = title.encode("utf-8") if isinstance(subtitle, unicode): subtitle = subtitle.encode("utf-8") if isinstance(summary, unicode): summary = summary.encode("utf-8") print title print subtitle print url EntryList(title=title, subtitle=subtitle, url=url, datatype="audio/mpeg", summary=summary, date=date, image=image) print i now_no = c * 20 + 1 trans.commit() except Exception: trans.rollback() raise def inportBackNumber01(): url = "http://www.galge.com/radio/galge/back01.html" html = urllib2.urlopen(url) print "Loading..." bs = BS(html.read()) print "Adding..." pat = re.compile(".*\.mp3$") for i in range(1,21): content = bs.findAll("a", {"name" : "%02d"%i})[0].findNext() title = content.h4.contents[0] try: subtitle = content.b.next.next.next except AttributeError: subtitle = None if not isinstance(subtitle,basestring): subtitle = None summary = content.h4.next.next summary = summary + "\n" +content.h4.next.next.next.next for atag in content.findAll("a"): url = atag.attrMap["href"] if pat.match(url): break date_src = urllib2.urlopen(url).headers.getheader('Last-Modified') if date_src: date = datetime.datetime.fromtimestamp(time.mktime( if date_src: date = datetime.datetime.fromtimestamp(time.mktime( email.utils.parsedate(date_src))) else: date = None if content.find("img",{"width":200}): image = content.find("img",{"width":200}).attrMap["src"] else: if content.parent.find("img",{"width":200}): image = content.parent.find("img",{"width":200}).attrMap["src"] else: image = None if isinstance(title, unicode): title = title.encode("utf-8") if isinstance(subtitle, unicode): subtitle = subtitle.encode("utf-8") if isinstance(summary, unicode): summary = summary.encode("utf-8") print title print url EntryList(title=title, subtitle=subtitle, url=url, datatype="audio/mpeg", summary=summary, date=date, image=image) print i if __name__ == "__main__": #importBackNumber(104, 110) toRSS(rss_path)
